apiVersion: batch/v1
kind: Job
metadata:
  name: ts-spatop-train-v8-full-test
  labels:
    jobgroup: spatop-job-train
spec:
  template:
    spec:
      securityContext:
        runAsUser: 0
        fsGroup: 0
      containers:
        - name: training
          image: gitlab-registry.nrp-nautilus.io/mquinnan/top-spanet:spatop-container
          command:
            - "/bin/bash"
            - "-c"
            - |
              set -e  # exit on error

              echo "Installing SPANet v2.3, tensorboard and rich..."
              pip install git+https://github.com/Alexanders101/SPANet.git@v2.3
              pip install tensorboard rich

              echo "Starting training job..."
              python -m spanet.train \
                -of /spatopvol/options_files/v8/spatop_v7_fulltest.json \
                -l /spatopvol/logs/ \
                -n spatop_v8_fulltest_output 
          volumeMounts:
            - name: dshm
              mountPath: /dev/shm
            - mountPath: /spatopvol
              name: spatopvol
            - name: git-repo
              mountPath: /opt/repo
          resources:
            limits:
              memory: 64Gi
              cpu: "16"
              nvidia.com/gpu: "1"
            requests:
              memory: 32Gi
              cpu: "8"
              nvidia.com/gpu: "1"
      initContainers:
        - name: init-clone-repo
          image: alpine/git
          args:
            - clone
            - --single-branch
            - https://github.com/quinnanm/SPAtop
            - -b
            - meldev
            - --recurse-submodules
            - /opt/repo/SPAtop
          volumeMounts:
            - name: git-repo
              mountPath: /opt/repo
      volumes:
        - name: dshm
          emptyDir:
            medium: Memory
            sizeLimit: 8Gi
        - name: git-repo
          emptyDir: {}
        - name: spatopvol
          persistentVolumeClaim:
            claimName: spatopvol
      # tolerations:
      #   - key: "nautilus.io/reservation"
      #     operator: "Equal"
      #     value: "spanet"
      #     effect: "NoSchedule"
      restartPolicy: Never
  backoffLimit: 5