:W&B5sM wf53czly; 9 /0.28.1+bc2cf24b1bd9fa3a1b6cbeb920087b832afc526f0.70.0[" tsjt4xbl1kfx wf53czly wf53czlyescheuller-uc-san-diego spatop-sweep" _wandb{}  batch_size2048  dropout0.23164927762604837  hidden_dim256 ' learning_rate0.0006017469678417099  num_embedding_layers10Bdpfx_vanilla_wf53czlybnqupkqclj%dp-spatop-sweepagent-vanilla-v2-h9cgs ԝ3 (Hhp"3.12.10*0.28.1j linux-x86_6495}; wf53czly*  _wandb {"runtime":0}  _runtime00b -Linux-6.8.0-134-generic-x86_64-with-glibc2.36CPython 3.12.10 ԝ*-of*>/data/spatop/options_files/sweep_fixed_dp/config_wf53czly.json*-l* /scratch/logs*-n*dpfx_vanilla_wf53czly2-m spanet.trainRdanielprimosch@gmail.comZ /scratch/logsb%dp-spatop-sweepagent-vanilla-v2-h9cgsr/opt/conda/bin/python 66pcipxu6dq54ytgp8jisagma0wtiuft <NVIDIA A100-SXM4-80GB /.ӽS NVIDIA A100-SXM4-80GB6"Ampere*(GPU-0c583aac-6f68-6140-7078-f37971b4b9bb13.2 66pcipxu6dq54ytgp8jisagma0wtiuft2  requirements.txtqW wf53czlyZF (H (H (hp"3.12.10*0.28.1j linux-x86_64Y wf53czlyZH (H (H (8hp"3.12.10*0.28.1j linux-x86_64뀡* wf53czlyb trainer/global_step2xlS\ wf53czlyZK (H (H(8hp"3.12.10*0.28.1j linux-x86_64ه// wf53czlyb*"trainer/global_step2ײK wf53czlyj: ՝*LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] %Ɠ wf53czlyj  םҿ ┏━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━┳━━━━━━━┓ ┃ ┃ Name ┃ Type ┃ Params ┃ Mode ┃ FLOPs ┃ ┡━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━╇━━━━━━━┩ │ 0 │ embedding │ MultiInputVectorEmbed… │ 3.4 M │ train │ 0 │ │ 1 │ encoder │ JetEncoder │ 5.3 M │ train │ 0 │ │ 2 │ branch_decoders │ ModuleList │ 75.8 M │ train │ 0 │ │ 3 │ regression_decoder │ RegressionDecoder │ 0 │ train │ 0 │ │ 4 │ classification_decoder │ ClassificationDecoder │ 0 │ train │ 0 │ │ │ other params │ n/a │ 12 │ n/a │ n/a │ └───┴────────────────────────┴────────────────────────┴────────┴───────┴───────┘ B \ wf53czlyj םTrainable params: 84.5 M Non-trainable params: 50 Total params: 84.5 M Total estimated model params size (MB): 337.922 Modules in train mode: 2299 Modules in eval mode: 0 Total FLOPs: 0 y9Sq wf53czlyj` םPwandb: WARNING Config item 'hidden_dim' was locked by 'sweep' (ignored update). H{ wf53czlyjj ם؁Zwandb: WARNING Config item 'num_embedding_layers' was locked by 'sweep' (ignored update). >}q wf53czlyj` םPwandb: WARNING Config item 'batch_size' was locked by 'sweep' (ignored update). %Bt wf53czlyjc םSwandb: WARNING Config item 'learning_rate' was locked by 'sweep' (ignored update). >n wf53czlyj] םMwandb: WARNING Config item 'dropout' was locked by 'sweep' (ignored update). 1 wf53czly*  transformer_dim256  transformer_dim_scale2.0  initial_embedding_dim256  position_embedding_dim256  num_encoder_layers4 ! num_branch_embedding_layers3  num_branch_encoder_layers3  num_jet_embedding_layers0  num_jet_encoder_layers2  num_detector_layers2  num_regression_layers3  num_classification_layers3  split_symmetric_attention1  num_attention_heads8 ! transformer_activation"gelu"  skip_connections1 ( "initial_embedding_skip_connections1  linear_block_type"GRU"  transformer_type"Gated"  linear_activation"gelu"  normalization "LayerNorm"  masking "Filling" linear_prelu_activationtrue # use_pairwise_interactionsfalse  num_pairwise_features4  pairwise_embedding_dim8  pairwise_input_source""  use_moefalse  num_experts8  num_experts_per_tok2  moe_loss_scale0.01 K event_info_file7"/data/spatop/event_files/v11/tt_hadronic_v7_full.yaml" F training_file4"/data/spatop/tpm70_wpm30_FB350_fixed/all_merged.h5"  validation_file""  testing_file""  normalize_features1  limit_to_num_jets0  balance_particles0  balance_jets0 ! balance_classificationsfalse  partial_events1  dataset_limit1.0  dataset_randomization0  train_validation_split0.95  limit_val_batches1.0  num_dataloader_workers0  mask_sequence_vectors1  combine_pair_loss"min"  optimizer"AdamW"  focal_gamma0.0  combinatorial_scale0.0 # learning_rate_warmup_epochs1.0  learning_rate_cycles1  assignment_loss_scale1.0  detection_loss_scale1.0  kl_loss_scale0.0  regression_loss_scale0.0 ! classification_loss_scale0.0 ! use_mass_classificationfalse  mass_classes[]  balance_lossestrue  l2_penalty0.0002  gradient_clip10.0 epochs50 num_gpu1  verbose_outputfalse  usable_gpus""  trial_time"" $ trial_output_dir"./test_output"Wב wf53czlyj ם/opt/conda/lib/python3.12/site-packages/pytorch_lightning/utilities/_pytree.py:21: `isinstance(treespec, LeafSpec)` is deprecated, use `isinstance(treespec, TreeSpec) and treespec.is_leaf()` instead. v6", line 198, in _run_module_as_main "ŭ wf53czlyj ߝ}jaP wf53czlyj? ߝ/ File "", line 88, in _run_code ~,! wf53czlyj ߝ-ay" wf53czlyjh ߝ耶X File "/opt/conda/lib/python3.12/site-packages/spanet/train.py", line 277, in 8# wf53czlyj ߝ A%$ wf53czlyj ߝ E% wf53czlyj4 ߝ̻$main(**parser.parse_args().__dict__)5D"& wf53czlyj ߝ ۑ/' wf53czlyj ߝȶ)'/u( wf53czlyjd ߝѶT File "/opt/conda/lib/python3.12/site-packages/spanet/train.py", line 207, in main D?) wf53czlyj ߝS%%* wf53czlyj ߝ 6]I+ wf53czlyj8 ߝ(trainer.fit(model, ckpt_path=checkpoint)#", wf53czlyj ߝ ]8- wf53czlyj ߝ6_. wf53czlyjx ߝڇh File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/trainer.py", line 584, in fit @6/ wf53czlyj ߝ蹷5%0 wf53czlyj ߝ uzA1 wf53czlyj0 ߝ÷ call._call_and_handle_interrupt(z\&"2 wf53czlyj ߝ؀̷ ?E 3 wf53czlyj ߝѷS4 wf53czlyj ߝַ{ File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/call.py", line 49, in _call_and_handle_interrupt I5 wf53czlyj ߝ%6 wf53czlyj ߝ ȹC7 wf53czlyj2 ߝ"return trainer_fn(*args, **kwargs)"8 wf53czlyj ߝ )e9 wf53czlyj ߝU6": wf53czlyj ߝ j0"; wf53czlyj ߝ w2"< wf53czlyj ߝ𚔸 KA"= wf53czlyj ߝȹ 9"> wf53czlyj ߝ져 YBu"? wf53czlyj ߝؤ v"@ wf53czlyj ߝ r"A wf53czlyj ߝ &d"} wf53czlyj ߝ^9"~ wf53czlyj ߝɲ^," wf53czlyj ߝ^#r# wf53czlyj ߝٺ^9# wf53czlyj ߝ^Jt# wf53czlyj ߝ^r/X# wf53czlyj ߝŻ^# wf53czlyj ߝɻ^ # wf53czlyj ߝϻ^p# wf53czlyj ߝӻ^W/# wf53czlyj ߝֻ^^jL# wf53czlyj ߝٻ^%2# wf53czlyj ߝܻ^k# wf53czlyj ߝ߻^o# wf53czlyj ߝ^^0# wf53czlyj ߝ^5-# wf53czlyj ߝȮ K{4  wf53czlyj ߝ^ȓ wf53czlyj ߝȇp File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/trainer.py", line 1123, in _run_stage UF  wf53czlyj ߝi& wf53czlyj ߝ P25 wf53czlyj# ߝself.fit_loop.run()t# wf53czlyj ߝ IZ  wf53czlyj ߝҬv~ wf53czlyjw ߝӱg File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/loops/fit_loop.py", line 217, in run <+$  wf53czlyj ߝҼdn& wf53czlyj ߝةټ )P0 wf53czlyj ߝself.advance()7Iuc# wf53czlyj ߝ a  wf53czlyj ߝ wf53czlyj{ ߝk File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/loops/fit_loop.py", line 469, in advance ER:  wf53czlyj ߝ{& wf53czlyj ߝ՛ ToI wf53czlyj7 ߝ'self.epoch_loop.run(self._data_fetcher)݌$# wf53czlyj ߝ   wf53czlyj ߝ"フ wf53czlyj ߝϲr File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 153, in run U  wf53czlyj ߝν8& wf53czlyj ߝԽ 4< wf53czlyj* ߝٽself.advance(data_fetcher)!# wf53czlyj ߝ YJY  wf53czlyj ߝ wf53czlyj ߝv File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 352, in advance   wf53czlyj ߝȍ> & wf53czlyj ߝ䓾 z wf53czlyjh ߝśXbatch_output = self.automatic_optimization.run(trainer.optimizers[0], batch_idx, kwargs)!# wf53czlyj ߝ jZ׳  wf53czlyj ߝ*U# wf53czlyj ߝ 1 # wf53czlyj ߝź &3# wf53czlyj ߝȔ n%# wf53czlyj ߝž [ 4# wf53czlyj ߝȾ !|# wf53czlyj ߝ̾ # wf53czlyj ߝҾ @ # wf53czlyj ߝ־ M7# wf53czlyj ߝܾ 䪗# wf53czlyj ߝ ǴG# wf53czlyj ߝ _ˇ# wf53czlyj ߝ 3d# wf53czlyj ߝ # wf53czlyj ߝ d7# wf53czlyj ߝА $}1# wf53czlyj ߝ n3# wf53czlyj ߝ Š# wf53czlyj ߝ΍ 1# wf53czlyj ߝ i_# wf53czlyj ߝ^Ɏ# wf53czlyj ߝȿ^Lܘ# wf53czlyj ߝο^ g# wf53czlyj ߝտ^8# wf53czlyj ߝۿ^:w# wf53czlyj ߝ߿^@# wf53czlyj ߝ^# wf53czlyj ߝ^^$# wf53czlyj ߝ^M}C# wf53czlyj ߝ^ H# wf53czlyj ߝ^U\# wf53czlyj ߝ^=G# wf53czlyj ߝ؊^ h# wf53czlyj ߝ^s# wf53czlyj ߝ^LP# wf53czlyj ߝ^8# wf53czlyj ߝ^OH0# wf53czlyj ߝ^JQ# wf53czlyj ߝ؞^# wf53czlyj ߝ^57# wf53czlyj ߝ^'<# wf53czlyj ߝ^{ # wf53czlyj ߝ^;@# wf53czlyj ߝ^# wf53czlyj ߝ^էF# wf53czlyj ߝȁ^)# wf53czlyj ߝ^J# wf53czlyj ߝ^?P# wf53czlyj ߝ^# wf53czlyj ߝ^W# wf53czlyj ߝ^x# wf53czlyj ߝ^# wf53czlyj ߝ^ , # wf53czlyj ߝЧ^b# wf53czlyj ߝВ^oج# wf53czlyj ߝت^]X# wf53czlyj ߝ^ǡ# wf53czlyj ߝ^]&# wf53czlyj ߝԩ^l*# wf53czlyj ߝ^ b# wf53czlyj ߝ^XR# wf53czlyj ߝи^y O# wf53czlyj ߝݻ^1# wf53czlyj ߝ^# wf53czlyj ߝЅ^3fd# wf53czlyj ߝ^#y # wf53czlyj ߝ^U2)# wf53czlyj ߝ^# wf53czlyj ߝ^d# wf53czlyj ߝ^@# wf53czlyj ߝ^;# wf53czlyj ߝп^ꌋ# wf53czlyj ߝ^UÓy# wf53czlyj ߝ^r# wf53czlyj ߝ^5# wf53czlyj ߝ^tsG# wf53czlyj ߝ^^ׯ# wf53czlyj ߝ^kn# wf53czlyj ߝȵ^p.x# wf53czlyj ߝ^j# wf53czlyj ߝ^# wf53czlyj ߝ螸^5n# wf53czlyj ߝ^# wf53czlyj ߝ^g# wf53czlyj ߝ^#r # wf53czlyj ߝ^hJ# wf53czlyj ߝ T'=  wf53czlyj ߝ& wf53czlyj ߝu File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 192, in run w  wf53czlyj ߝS@7& wf53czlyj ߝؔ aJ wf53czlyj8 ߝ(self._optimizer_step(batch_idx, closure)# wf53czlyj ߝ 9  wf53czlyj ߝЙa wf53czlyj ߝ File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 270, in _optimizer_step S   wf53czlyj ߝƈ16D& wf53czlyj ߝ萍 !lC wf53czlyj1 ߝЌ!call._call_lightning_module_hook(n # wf53czlyj ߝ Ke  wf53czlyj ߝ๜ wf53czlyj ߝ} File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/trainer/call.py", line 177, in _call_lightning_module_hook ݢ  wf53czlyj ߝi& wf53czlyj ߝ jJ> wf53czlyj, ߝoutput = fn(*args, **kwargs)񰊡# wf53czlyj ߝ Q  wf53czlyj ߝzuD# wf53czlyj ߝ +v~# wf53czlyj ߝ K~# wf53czlyj ߝ i# wf53czlyj ߝ؟ # wf53czlyj ߝ *# wf53czlyj ߝ +# wf53czlyj ߝ # wf53czlyj ߝ *?# wf53czlyj ߝ |}i# wf53czlyj ߝ K&V# wf53czlyj ߝ 2k$# wf53czlyj ߝ X## wf53czlyj ߝଢ଼ ]# wf53czlyj ߝ^ʦ# wf53czlyj ߝѫ^$I# wf53czlyj ߝϱ^r# wf53czlyj ߝ׶^柍# wf53czlyj ߝ^i## wf53czlyj ߝ^l# wf53czlyj ߝȖ^\# wf53czlyj ߝ^`Ve># wf53czlyj ߝ^R># wf53czlyj ߝ^ܚ# wf53czlyj ߝ^a6# wf53czlyj ߝ؍^4†^# wf53czlyj ߝ^ήH# wf53czlyj ߝ^>k# wf53czlyj ߝ^d{K# wf53czlyj ߝ^bvP# wf53czlyj ߝ荈^ v# wf53czlyj ߝ^̍# wf53czlyj ߝŒ^pc# wf53czlyj ߝ p{  wf53czlyj ߝڠ- wf53czlyj ߝp File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/core/module.py", line 1368, in optimizer_step ,k  wf53czlyj ߝ&& wf53czlyj ߝ ,K wf53czlyj9 ߝ)optimizer.step(closure=optimizer_closure)-# wf53czlyj ߝ "{  wf53czlyj ߝ\I wf53czlyjx ߝӕh File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/core/optimizer.py", line 154, in step 1  wf53czlyj ߝԱ ՚& wf53czlyj ߝ ]q wf53czlyj_ ߝOstep_output = self._strategy.optimizer_step(self._optimizer, closure, **kwargs)+# wf53czlyj ߝ   wf53czlyj ߝi# wf53czlyj ߝ l# wf53czlyj ߝ # wf53czlyj ߝ b# wf53czlyj ߝ # wf53czlyj ߝ p# wf53czlyj ߝ 3X# wf53czlyj ߝ 9# wf53czlyj ߝ =w!# wf53czlyj ߝ l# wf53czlyj ߝ Zr# wf53czlyj ߝ \# wf53czlyj ߝȂ  # wf53czlyj ߝ &o(# wf53czlyj ߝ B # wf53czlyj ߝ # wf53czlyj ߝϽ p# wf53czlyj ߝ C# wf53czlyj ߝ d # wf53czlyj ߝ^ӛ# wf53czlyj ߝ^.^# wf53czlyj ߝر^C# wf53czlyj ߝ^F# wf53czlyj ߝ^ C# wf53czlyj ߝȘ^&# wf53czlyj ߝ^# wf53czlyj ߝؿ^Z# wf53czlyj ߝ^wZ# wf53czlyj ߝŠ^2O# wf53czlyj ߝఐ^## wf53czlyj ߝ^ v# wf53czlyj ߝ^DY# wf53czlyj ߝ衢^S`# wf53czlyj ߝ^#D8# wf53czlyj ߝ^|# wf53czlyj ߝ^L# wf53czlyj ߝ^TR# wf53czlyj ߝ^# wf53czlyj ߝ^k# wf53czlyj ߝ^_pY# wf53czlyj ߝ^k=z# wf53czlyj ߝ^Vs# wf53czlyj ߝ^2=# wf53czlyj ߝ^0k# wf53czlyj ߝȏ^# wf53czlyj ߝ^c{# wf53czlyj ߝ^F3# wf53czlyj ߝ^# wf53czlyj ߝ^7f# wf53czlyj ߝ݋^c# wf53czlyj ߝ^5# wf53czlyj ߝ^FQ[# wf53czlyj ߝ^ # wf53czlyj ߝǯ^k_# wf53czlyj ߝ^] # wf53czlyj ߝ^-Z# wf53czlyj ߝȖ^v7# wf53czlyj ߝȊ^# wf53czlyj ߝ^;# wf53czlyj ߝ^J!# wf53czlyj ߝ^v # wf53czlyj ߝ^=YO# wf53czlyj ߝ^DQf# wf53czlyj ߝ^A# wf53czlyj ߝ^Z?# wf53czlyj ߝ^cD# wf53czlyj ߝ^T@# wf53czlyj ߝ^ד# wf53czlyj ߝ^Q(h# wf53czlyj ߝ^z# wf53czlyj ߝ^HG# wf53czlyj ߝ^1HM# wf53czlyj ߝи Qq]  wf53czlyj ߝ6߅ wf53czlyjs ߝػc File "/opt/conda/lib/python3.12/site-packages/torch/optim/lr_scheduler.py", line 166, in wrapper   wf53czlyj ߝ}& wf53czlyj ߝ PZ wf53czlyjH ߝе8return func.__get__(opt, opt.__class__)(*args, **kwargs)qf# wf53czlyj ߝλ   wf53czlyj ߝ9b# wf53czlyj ߝ # wf53czlyj ߝ fmM# wf53czlyj ߝ o# wf53czlyj ߝ l=}G# wf53czlyj ߝ؞ 6# wf53czlyj ߝ a@# wf53czlyj ߝ 8# wf53czlyj ߝ # wf53czlyj ߝѻ^n`# wf53czlyj ߝ^e"# wf53czlyj ߝ^0!s# wf53czlyj ߝ^ Yo# wf53czlyj ߝ^EăC# wf53czlyj ߝ^E# wf53czlyj ߝ^ # wf53czlyj ߝ^[# wf53czlyj ߝ^!џ# wf53czlyj ߝ^m3r# wf53czlyj ߝ^+$# wf53czlyj ߝ 6v  wf53czlyj ߝ wf53czlyjq ߝa File "/opt/conda/lib/python3.12/site-packages/torch/optim/optimizer.py", line 80, in _use_grad 5W  wf53czlyj ߝp}1& wf53czlyj ߝ Q= wf53czlyj+ ߝМret = func(*args, **kwargs)2S# wf53czlyj ߝ \~  wf53czlyj ߝt# wf53czlyj ߝ E# wf53czlyj ߝ½ z# wf53czlyj ߝ AS# wf53czlyj ߝ s)# wf53czlyj ߝ # wf53czlyj ߝЌ R # wf53czlyj ߝ &q# wf53czlyj ߝ ~#/# wf53czlyj ߝ gi# wf53czlyj ߝ j# wf53czlyj ߝ^Ύ# wf53czlyj ߝ^# wf53czlyj ߝ^ƈ# wf53czlyj ߝ^8# wf53czlyj ߝ^36" wf53czlyj о " wf53czlyj 蛠 5" wf53czlyj  C$" wf53czlyj  " wf53czlyj  ѾT" wf53czlyj  lp" wf53czlyj  nacc" wf53czlyj  " wf53czlyj  s" wf53czlyj Х " wf53czlyj  zN" wf53czlyj  Kl" wf53czlyj  z"" wf53czlyj  e>" wf53czlyj  H`d" wf53czlyj Ȓ C;" wf53czlyj  u " wf53czlyj  )jw" wf53czlyj  z!Z" wf53czlyj  b" wf53czlyj ^J" wf53czlyj Ϟ^" wf53czlyj 茢^;E" wf53czlyj ^O" wf53czlyj Ȩ^H" wf53czlyj ^v" wf53czlyj ଴^>a" wf53czlyj ^!" wf53czlyj ^F" wf53czlyj  4j wf53czlyj  ~h wf53czlyj z File "/opt/conda/lib/python3.12/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 146, in __call__ ճ wf53czlyj  4% wf53czlyj  ,QM wf53czlyj; ,self._result = self.closure(*args, **kwargs) 8)" wf53czlyj  Tö wf53czlyj  [" wf53czlyj Ċ " wf53czlyj ǎ " wf53czlyj л c?D" wf53czlyj Ζ k" wf53czlyj  " wf53czlyj  \I" wf53czlyj  3n" wf53czlyj  Y" wf53czlyj  11" wf53czlyj Ⱥ " wf53czlyj ô kCs" wf53czlyj  [" wf53czlyj  ;" wf53czlyj  " wf53czlyj  Y8.) _wandbruntime13  _runtime13+(