{"DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.freed":324862511271936,"DeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.peak":189288448,"lr-AdamW/pg2":0.00046387358454902443,"DeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.peak":2712961024,"DeviceStatsMonitor.on_train_batch_start/allocation.all.peak":11887,"DeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.freed":15799443739300,"DeviceStatsMonitor.on_train_batch_start/active.large_pool.current":2,"DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.current":2,"DeviceStatsMonitor.on_train_batch_end/inactive_split.all.freed":122492572,"DeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.freed":309023426547712,"DeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.peak":248886712,"DeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.peak":253755392,"DeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.peak":248886712,"DeviceStatsMonitor.on_train_batch_end/active.large_pool.allocated":119585222,"DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.allocated":3021996032,"DeviceStatsMonitor.on_train_batch_end/segment.large_pool.freed":0,"DeviceStatsMonitor.on_train_batch_start/num_oom_rejections":0,"DeviceStatsMonitor.on_train_batch_start/oversize_segments.current":0,"DeviceStatsMonitor.on_train_batch_start/segment.small_pool.freed":0,"DeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.freed":309733926734848,"DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.allocated":324844569358336,"DeviceStatsMonitor.on_train_batch_end/inactive_split.all.allocated":122492819,"DeviceStatsMonitor.on_train_batch_start/allocation.all.current":9658,"DeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.peak":189288448,"DeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.current":87429632,"DeviceStatsMonitor.on_train_batch_end/oversize_segments.allocated":0,"DeviceStatsMonitor.on_train_batch_end/active_bytes.all.peak":2940455936,"DeviceStatsMonitor.on_train_batch_start/num_device_free":0,"DeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.allocated":51617554,"DeviceStatsMonitor.on_train_batch_start/active_bytes.all.freed":324844464889344,"DeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.allocated":301327663476832,"DeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.allocated":15838292534272,"DeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.current":17039360,"DeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.allocated":301344403649152,"loss/SRqqt1/assignment_loss":0.646479534978668,"DeviceStatsMonitor.on_train_batch_start/inactive_split.all.allocated":122485951,"DeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.current":2,"DeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.current":17039360,"DeviceStatsMonitor.on_train_batch_end/allocation.all.current":9672,"DeviceStatsMonitor.on_train_batch_start/allocation.small_pool.peak":11816,"DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.current":3021996032,"loss/FBt2/detection_loss":1.2967494487977371,"DeviceStatsMonitor.on_train_batch_start/segment.large_pool.freed":0,"DeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.allocated":253755392,"DeviceStatsMonitor.on_train_batch_start/allocation.small_pool.current":9656,"DeviceStatsMonitor.on_train_batch_end/oversize_allocations.peak":0,"DeviceStatsMonitor.on_train_batch_start/active.small_pool.peak":11816,"DeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.current":87436800,"DeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.allocated":51620485,"DeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.freed":309006259784704,"DeviceStatsMonitor.on_train_batch_end/active_bytes.all.allocated":324862615748096,"DeviceStatsMonitor.on_train_batch_end/oversize_segments.peak":0,"DeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.allocated":326019219576832,"DeviceStatsMonitor.on_train_batch_start/oversize_allocations.freed":0,"DeviceStatsMonitor.on_train_batch_start/segment.large_pool.peak":130,"DeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.current":24903680,"DeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.allocated":2768240640,"DeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.allocated":16267145026048,"DeviceStatsMonitor.on_train_batch_start/segment.small_pool.allocated":121,"DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.current":9670,"loss/total_loss":15.818564673533654,"DeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.current":2,"DeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.current":87429632,"DeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.freed":15839084724224,"DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.peak":2871874888,"loss/FRt2/assignment_loss":2.0810435178379216,"DeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.allocated":2768240640,"DeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.allocated":253755392,"DeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.allocated":16268060821504,"DeviceStatsMonitor.on_train_batch_start/oversize_segments.peak":0,"DeviceStatsMonitor.on_train_batch_end/max_split_size":-1,"DeviceStatsMonitor.on_train_batch_end/oversize_allocations.freed":0,"DeviceStatsMonitor.on_train_batch_end/segment.all.peak":251,"DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.allocated":190097562,"DeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.peak":538,"DeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.current":253755392,"DeviceStatsMonitor.on_train_batch_end/active_bytes.all.freed":324862511271936,"_timestamp":1.785803291806067e+09,"DeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.current":2768240640,"DeviceStatsMonitor.on_train_batch_end/active.small_pool.freed":190087892,"DeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.current":17039360,"DeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.current":246,"DeviceStatsMonitor.on_train_batch_start/segment.all.peak":251,"DeviceStatsMonitor.on_train_batch_end/oversize_allocations.current":0,"DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.peak":3021996032,"DeviceStatsMonitor.on_train_batch_end/segment.large_pool.peak":130,"DeviceStatsMonitor.on_train_batch_start/active.large_pool.freed":119578577,"DeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.freed":15839084724224,"DeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.peak":145651200,"DeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.peak":2712961024,"DeviceStatsMonitor.on_train_batch_start/inactive_split.all.freed":122485703,"DeviceStatsMonitor.on_train_batch_start/active.all.peak":11887,"DeviceStatsMonitor.on_train_batch_start/active.all.freed":309655946,"DeviceStatsMonitor.on_train_batch_end/segment.small_pool.allocated":121,"lr-AdamW/pg1":0.00046387358454902443,"DeviceStatsMonitor.on_train_batch_end/active_bytes.all.current":104476160,"DeviceStatsMonitor.on_train_batch_start/allocation.large_pool.allocated":119578579,"DeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.freed":15838205104640,"DeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.freed":16267934348800,"DeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.current":253755392,"DeviceStatsMonitor.on_train_batch_end/active.large_pool.freed":119585220,"DeviceStatsMonitor.on_train_batch_end/inactive_split.all.peak":548,"DeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.current":24903680,"_step":1439,"loss/SRqqt2/detection_loss":1.25928864729549,"DeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.allocated":309751158755328,"DeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.allocated":309023443587072,"DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.peak":11816,"DeviceStatsMonitor.on_train_batch_start/segment.large_pool.current":130,"DeviceStatsMonitor.on_train_batch_start/active.small_pool.allocated":190087025,"DeviceStatsMonitor.on_train_batch_start/num_device_alloc":251,"_runtime":8011,"DeviceStatsMonitor.on_train_batch_start/segment.small_pool.current":121,"DeviceStatsMonitor.on_train_batch_end/segment.all.current":251,"DeviceStatsMonitor.on_train_batch_start/segment.all.current":251,"DeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.allocated":309733951638528,"loss/FBt1/detection_loss":1.1507717702123854,"DeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.peak":250523136,"DeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.allocated":326001096664576,"DeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.current":2768240640,"DeviceStatsMonitor.on_train_batch_end/active_bytes.large_pool.freed":309023426547712,"DeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.freed":309751133851648,"DeviceStatsMonitor.on_train_batch_start/active_bytes.all.current":104468992,"DeviceStatsMonitor.on_train_batch_start/num_sync_all_streams":2,"DeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.peak":2768240640,"DeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.freed":309006259784704,"DeviceStatsMonitor.on_train_batch_start/oversize_allocations.current":0,"DeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.current":17039360,"loss/FBt2/assignment_loss":1.5634643741708924e-06,"DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.peak":2940455936,"DeviceStatsMonitor.on_train_batch_start/allocation.all.allocated":309665604,"DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.allocated":119585222,"DeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.current":157675008,"loss/SRqqt2/assignment_loss":0.6820241181708094,"DeviceStatsMonitor.on_train_batch_start/oversize_allocations.peak":0,"loss/FRt1/assignment_loss":2.778220461308956,"DeviceStatsMonitor.on_train_batch_start/segment.all.freed":0,"DeviceStatsMonitor.on_train_batch_start/allocation.large_pool.peak":1291,"DeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.large_pool.peak":86278144,"DeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.allocated":309006276824064,"DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.freed":324844464889344,"DeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.all.freed":326000938989568,"DeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.peak":250523136,"DeviceStatsMonitor.on_train_batch_end/reserved_bytes.small_pool.freed":0,"DeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.current":132771328,"DeviceStatsMonitor.on_train_batch_end/num_oom_rejections":0,"DeviceStatsMonitor.on_train_batch_start/inactive_split.all.peak":548,"DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.freed":119585220,"DeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.freed":15800321212522,"DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.freed":0,"DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.freed":317127090176772,"DeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.freed":326019068200448,"DeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.allocated":15799529542424,"DeviceStatsMonitor.on_train_batch_start/active.all.current":9658,"DeviceStatsMonitor.on_train_batch_end/segment.small_pool.freed":0,"DeviceStatsMonitor.on_train_batch_end/segment.large_pool.current":130,"DeviceStatsMonitor.on_train_batch_start/segment.small_pool.peak":121,"DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.current":102842596,"DeviceStatsMonitor.on_train_batch_start/max_split_size":-1,"trainer/global_step":17999,"DeviceStatsMonitor.on_train_batch_end/allocated_bytes.small_pool.allocated":15839172161024,"DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.current":102842484,"DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.allocated":324862615748096,"DeviceStatsMonitor.on_train_batch_start/allocation.small_pool.allocated":190087025,"DeviceStatsMonitor.on_train_batch_end/active.small_pool.allocated":190097562,"DeviceStatsMonitor.on_train_batch_start/active.large_pool.peak":1291,"DeviceStatsMonitor.on_train_batch_start/active_bytes.all.allocated":324844569358336,"DeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.freed":0,"DeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.freed":15838205104640,"DeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.large_pool.peak":86278144,"loss/SRqqt1/detection_loss":1.1250284381893774,"DeviceStatsMonitor.on_train_batch_end/allocation.large_pool.peak":1291,"DeviceStatsMonitor.on_train_batch_end/allocation.all.allocated":309682784,"DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.freed":317144707822314,"DeviceStatsMonitor.on_train_batch_end/allocated_bytes.all.current":104476160,"DeviceStatsMonitor.on_train_batch_start/oversize_segments.allocated":0,"DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.allocated":3021996032,"loss/FRt2/detection_loss":2.114648920328667,"DeviceStatsMonitor.on_train_batch_start/oversize_allocations.allocated":0,"DeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.freed":301327646437472,"DeviceStatsMonitor.on_train_batch_end/inactive_split.all.current":247,"DeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.current":17039360,"DeviceStatsMonitor.on_train_batch_end/allocation.all.peak":11887,"DeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.current":85803236,"DeviceStatsMonitor.on_train_batch_end/active.all.current":9672,"DeviceStatsMonitor.on_train_batch_start/reserved_bytes.large_pool.freed":0,"DeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.allocated":70872334,"DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.peak":2871874888,"epoch":0,"DeviceStatsMonitor.on_train_batch_start/allocation.small_pool.freed":190077369,"DeviceStatsMonitor.on_train_batch_start/oversize_segments.freed":0,"DeviceStatsMonitor.on_train_batch_start/segment.all.allocated":251,"DeviceStatsMonitor.on_train_batch_start/allocation.large_pool.freed":119578577,"DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.current":104468992,"DeviceStatsMonitor.on_train_batch_end/active.small_pool.peak":11816,"loss/FRt1/detection_loss":2.6843082529492674,"DeviceStatsMonitor.on_train_batch_end/allocated_bytes.large_pool.allocated":309023443587072,"DeviceStatsMonitor.on_train_batch_end/requested_bytes.all.allocated":317144810664910,"DeviceStatsMonitor.on_train_batch_end/num_device_alloc":251,"DeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.freed":16267012254720,"DeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.peak":2712961024,"DeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.freed":70872332,"DeviceStatsMonitor.on_train_batch_start/inactive_split_bytes.small_pool.peak":145651200,"DeviceStatsMonitor.on_train_batch_end/active.all.freed":309673112,"DeviceStatsMonitor.on_train_batch_start/active.small_pool.freed":190077369,"DeviceStatsMonitor.on_train_batch_end/allocation.all.freed":309673112,"DeviceStatsMonitor.on_train_batch_start/active.small_pool.current":9656,"DeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.freed":301344386609792,"DeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.peak":2712961024,"DeviceStatsMonitor.on_train_batch_end/inactive_split.large_pool.peak":18,"DeviceStatsMonitor.on_train_batch_start/allocated_bytes.large_pool.allocated":309006276824064,"DeviceStatsMonitor.on_train_batch_end/segment.small_pool.current":121,"DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.peak":3021996032,"DeviceStatsMonitor.on_train_batch_start/active_bytes.all.peak":2940455936,"DeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.allocated":70868397,"DeviceStatsMonitor.on_train_batch_start/active.large_pool.allocated":119578579,"DeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.freed":0,"DeviceStatsMonitor.on_train_batch_start/reserved_bytes.all.freed":0,"DeviceStatsMonitor.on_train_batch_end/segment.all.freed":0,"DeviceStatsMonitor.on_train_batch_start/active_bytes.small_pool.peak":250523136,"DeviceStatsMonitor.on_train_batch_end/segment.all.allocated":251,"DeviceStatsMonitor.on_train_batch_start/allocated_bytes.all.peak":2940455936,"DeviceStatsMonitor.on_train_batch_start/requested_bytes.large_pool.peak":2645753856,"DeviceStatsMonitor.on_train_batch_start/allocation.all.freed":309655946,"DeviceStatsMonitor.on_train_batch_start/allocation.large_pool.current":2,"DeviceStatsMonitor.on_train_batch_end/active.large_pool.peak":1291,"DeviceStatsMonitor.on_train_batch_start/requested_bytes.all.allocated":317127193019256,"loss/FBt1/assignment_loss":0,"DeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.peak":250523136,"DeviceStatsMonitor.on_train_batch_start/allocated_bytes.small_pool.allocated":15838292534272,"DeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.freed":51620240,"DeviceStatsMonitor.on_train_batch_end/num_alloc_retries":0,"DeviceStatsMonitor.on_train_batch_start/active.all.allocated":309665604,"DeviceStatsMonitor.on_train_batch_start/inactive_split.small_pool.freed":51617308,"DeviceStatsMonitor.on_train_batch_end/oversize_allocations.allocated":0,"DeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.peak":18,"DeviceStatsMonitor.on_train_batch_end/num_sync_all_streams":2,"DeviceStatsMonitor.on_train_batch_end/active.all.peak":11887,"DeviceStatsMonitor.on_train_batch_end/oversize_segments.current":0,"DeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.all.current":151376384,"DeviceStatsMonitor.on_train_batch_start/active_bytes.large_pool.current":17039360,"DeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.current":87436800,"DeviceStatsMonitor.on_train_batch_start/requested_bytes.small_pool.current":85803124,"_wandb":{"runtime":8011},"DeviceStatsMonitor.on_train_batch_end/num_device_free":0,"DeviceStatsMonitor.on_train_batch_start/inactive_split.all.current":248,"DeviceStatsMonitor.on_train_batch_end/allocation.small_pool.freed":190087892,"DeviceStatsMonitor.on_train_batch_end/active.all.allocated":309682784,"DeviceStatsMonitor.on_train_batch_end/inactive_split_bytes.small_pool.current":126472704,"DeviceStatsMonitor.on_train_batch_start/num_alloc_retries":0,"DeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.peak":538,"DeviceStatsMonitor.on_train_batch_end/segment.large_pool.allocated":130,"DeviceStatsMonitor.on_train_batch_start/reserved_bytes.small_pool.peak":253755392,"DeviceStatsMonitor.on_train_batch_end/oversize_segments.freed":0,"DeviceStatsMonitor.on_train_batch_start/inactive_split.large_pool.freed":70868395,"DeviceStatsMonitor.on_train_batch_end/requested_bytes.small_pool.allocated":15800407015758,"DeviceStatsMonitor.on_train_batch_start/num_ooms":0,"DeviceStatsMonitor.on_train_batch_end/active.large_pool.current":2,"DeviceStatsMonitor.on_train_batch_end/requested_bytes.large_pool.peak":2645753856,"DeviceStatsMonitor.on_train_batch_end/inactive_split.small_pool.current":245,"DeviceStatsMonitor.on_train_batch_end/reserved_bytes.all.current":3021996032,"DeviceStatsMonitor.on_train_batch_end/num_ooms":0,"DeviceStatsMonitor.on_train_batch_start/segment.large_pool.allocated":130,"DeviceStatsMonitor.on_train_batch_end/active_bytes.small_pool.allocated":15839172161024,"DeviceStatsMonitor.on_train_batch_end/segment.small_pool.peak":121,"DeviceStatsMonitor.on_train_batch_end/active.small_pool.current":9670,"DeviceStatsMonitor.on_train_batch_end/reserved_bytes.large_pool.peak":2768240640}