#
# Example slurm.conf file generated by DeepOps. 
#
# Slurm provides configurator.html
# (in doc/html) to build a configuration file customized
# for your environment.
#
# See the slurm.conf man page for more information.

# Define a name for the cluster
ClusterName=deepops

# Configure the controllers
SlurmctldHost=dgxa100
StateSaveLocation=/var/spool/slurm/ctld
EnforcePartLimits=ALL

# Basic configuration
SlurmUser=slurm
SlurmctldPort=6817
SlurmdPort=6818
AuthType=auth/munge
SlurmdSpoolDir=/var/spool/slurm/d
SwitchType=switch/none
SlurmctldPidFile=/var/run/slurmctld.pid
SlurmdPidFile=/var/run/slurmd.pid
ProctrackType=proctrack/cgroup
PluginDir=/usr/local/lib/slurm
PropagateResourceLimitsExcept=MEMLOCK
MaxMemPerCPU=4000

# Basic job behavior
ReturnToService=1
RebootProgram="/bin/systemctl reboot"
ResumeTimeout=900

# Use PMIX as our default MPI configuration
MpiDefault=pmix

# Prolog/epilog config
PrologFlags=Alloc,Serial,Contain
Prolog=/etc/slurm/prolog.sh
Epilog=/etc/slurm/epilog.sh

#Submit config
JobSubmitPlugins=lua

# Health checking
HealthCheckProgram=/usr/sbin/nhc
HealthCheckInterval=300
HealthCheckNodeState=IDLE

# Mail program to use
MailProg=/usr/bin/s-nail

TaskPlugin=affinity,cgroup

# TIMERS
SlurmctldTimeout=120
SlurmdTimeout=300
InactiveLimit=0
MinJobAge=300
KillWait=30
Waittime=0

# SCHEDULING
SchedulerType=sched/backfill
#SchedulerAuth=
SelectType=select/cons_tres
SelectTypeParameters=CR_Core_Memory,CR_CORE_DEFAULT_DIST_BLOCK,CR_ONE_TASK_PER_CORE
PriorityType=priority/multifactor

# LOGGING
SlurmctldDebug=3
SlurmctldLogFile=/var/log/slurm/slurmctld.log
SlurmdDebug=3
SlurmdLogFile=/var/log/slurm/slurmd.log
JobCompType=jobcomp/none
#JobCompLoc=

# ACCOUNTING
JobAcctGatherType=jobacct_gather/none
JobAcctGatherFrequency=30
AccountingStorageTRES=gres/gpu,gres/gpu:A100.80gb,gres/gpu:3g.40gb,gres/gpu:2g.20gb,gres/gpu:1g.10gb
#
#DebugFlags=CPU_Bind,gres
AccountingStorageType=accounting_storage/slurmdbd
AccountingStorageHost=dgxa100
#AccountingStorageLoc=
AccountingStorageEnforce=associations,limits,qos
AccountingStorageUser=slurm
AccountingStoragePass=/var/run/munge/munge.socket.2

# COMPUTE NODE
GresTypes=gpu

NodeName=dgxa100 CPUs=124 Boards=1 SocketsPerBoard=1 CoresPerSocket=62 ThreadsPerCore=2 RealMemory=500000 Gres=gpu:1g.10gb:10,gpu:2g.20gb:4,gpu:3g.40gb:1,gpu:A100.80gb:1

PartitionName=interactive10 Nodes=dgxa100 Default=NO MaxTime=2:0:0 State=UP DefCpuPerGPU=4 DefMemPerCPU=4000
PartitionName=prod10 Priority=10000 Nodes=dgxa100 Default=NO MaxTime=24:0:0 DefaultTime=4:0:0 State=UP DefCpuPerGPU=4 DefMemPerCPU=4000
PartitionName=prod20 Priority=4000 Nodes=dgxa100 Default=NO MaxTime=24:0:0 DefaultTime=4:0:0 State=UP DefCpuPerGPU=8 DefMemPerCPU=4000
PartitionName=prod40 Priority=1000 Nodes=dgxa100 Default=NO MaxTime=24:0:0 DefaultTime=4:0:0 State=UP DefCpuPerGPU=16 DefMemPerCPU=4000
PartitionName=prod80 Priority=1000 Nodes=dgxa100 Default=NO MaxTime=24:0:0 DefaultTime=4:0:0 State=UP DefCpuPerGPU=32 DefMemPerCPU=4000