# Example slurm.conf file. Please run configurator.html
# (in doc/html) to build a configuration file customized
# for your environment.
#
#
# slurm.conf file generated by configurator.html.
#
# See the slurm.conf man page for more information.
#
ClusterName=I5
ControlMachine=blue51

BackupController=blue52

#Trigger job script
RebootProgram=/home/root/scripts/reconfig_node.sh

# LBNL Node Health Check (NHC)
HealthCheckProgram=/usr/sbin/nhc
HealthCheckInterval=3600
HealthCheckNodeState=ANY

# Premption settings
PreemptType=preempt/partition_prio
PreemptMode=gang,suspend
#
SlurmUser=slurm
#SlurmdUser=root
SlurmctldPort=6817
SlurmdPort=6818
AuthType=auth/munge
#JobCredentialPrivateKey=
#JobCredentialPublicCertificate=
StateSaveLocation=/mainfs/local/software/slurm/default/statesave
SlurmdSpoolDir=/tmp/slurmd
SwitchType=switch/none
MpiDefault=openmpi
SlurmctldPidFile=/var/run/slurmctld.pid
SlurmdPidFile=/var/run/slurmd.pid
ProctrackType=proctrack/cgroup
#PluginDir=
JobSubmitPlugins=lua
CacheGroups=0
#FirstJobId=
ReturnToService=1
#MaxJobCount=
#PlugStackConfig=
#PropagatePrioProcess=
#PropagateResourceLimits=
PropagateResourceLimitsExcept=ALL
PrologFlags=contain
#Prolog=
Epilog=/etc/slurm/slurm.epilog.clean
#SrunProlog=
#SrunEpilog=
TaskProlog=/etc/slurm/taskprolog
#TaskEpilog=
TaskPlugin=affinity,cgroup
#TrackWCKey=no
#TreeWidth=50
##TmpFS=/tmp
#UsePAM=1
#
# TIMERS
SlurmctldTimeout=60
SlurmdTimeout=300
InactiveLimit=0
MinJobAge=300
KillWait=600
Waittime=0
MessageTimeout=60
#
# SCHEDULING
JobRequeue=0
SchedulerType=sched/backfill
# 28/3/2019 change
# Based on https://slurm.schedmd.com/high_throughput.html
#SchedulerParameters=pack_serial_at_end,bf_window=3600,bf_resolution=600,bf_max_job_user=4,bf_continue
SchedulerParameters=assoc_limit_continue,batch_sched_delay=20,bf_continue,bf_interval=300,bf_min_age_reserve=10800,bf_window=3600,bf_resolution=600,bf_yield_interval=1000000,partition_job_depth=500,sched_max_job_start=200,sched_min_interval=2000000
#SchedulerAuth=
#SchedulerPort=
#SchedulerRootFilter=
SelectType=select/cons_res
#SelectTypeParameters=CR_CPU
SelectTypeParameters=CR_Core
FastSchedule=1
PriorityFavorSmall=NO
PriorityFlags=SMALL_RELATIVE_TO_TIME,ACCRUE_ALWAYS,FAIR_TREE
PriorityType=priority/multifactor
PriorityDecayHalfLife=7-0
#PriorityUsageResetPeriod=MONTHLY
#PriorityWeightFairshare=100000
#PriorityWeightAge=1000
#PriorityWeightPartition=10000
#PriorityWeightJobSize=100000
#PriorityWeightQOS=2000
#PriorityMaxAge=3-0
#
# After discussing our situation with SchedMD..
#
PriorityWeightFairshare=1000000
PriorityWeightAge=100000
PriorityWeightPartition=1000
#PriorityWeightJobSize=100000
PriorityWeightJobSize=10000000
PriorityWeightQOS=10000
# 28/3/2019 -- changed from 7 to 2 days
# 01/4/2019 -- changed from 2 to 1 days
PriorityMaxAge=1-0
#
# LOGGING
SlurmctldDebug=3
DebugFlags=Backfill
SlurmctldLogFile=/var/log/slurm/slurmctl.log
SlurmdDebug=3
SlurmdLogFile=/var/log/slurm/slurmd.log
JobCompType=jobcomp/none
#JobCompLoc=
#
# ACCOUNTING
JobAcctGatherType=jobacct_gather/linux
##JobAcctGatherFrequency=30
#


AccountingStorageType=accounting_storage/slurmdbd
AccountingStorageHost=10.13.39.203
AccountingStorageLoc=slurm_acct_db
AccountingStoragePass=Hocarj=onIbAfnoTwig6
AccountingStorageUser=slurm
AccountingStoragePort=6819
AccountingStoreJobComment=Yes
AccountingStorageEnforce=associations,limits,qos
AccountingStorageTRES=gres/gpu
#
# Health Check
#HealthCheckProgram=/usr/sbin/nhc
#HealthCheckInterval=300
#
GresTypes=gpu
# COMPUTE NODES
NodeName=red[001-490] Procs=40 Sockets=2 CoresPerSocket=20 ThreadsPerCore=1 RealMemory=172000 State=UNKNOWN
NodeName=orange03 Procs=48 Sockets=8 CoresPerSocket=6 ThreadsPerCore=1 RealMemory=1018990 State=UNKNOWN

NodeName=indigo[51-60] Gres=gpu:volta:2 Procs=40 Sockets=2 CoresPerSocket=20 ThreadsPerCore=1 RealMemory=190000 State=UNKNOWN
# hyperthreading is on..
NodeName=pink[51-60] Gres=gpu:gtx1080ti:4 Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=HyperThread RealMemory=120000 State=UNKNOWN
NodeName=linuxvm[1-2] Procs=10 Sockets=10 CoresPerSocket=1 ThreadsPerCore=1 RealMemory=192000 Feature=dcv2,vnc

#
# Partition Configurations
#
PartitionName=DEFAULT State=UP

# general queues
PartitionName=batch  nodes=red[001-464] ExclusiveUser=YES Default=Yes MaxCPUsPerNode=40 DefaultTime=02:00:00 MaxTime=60:00:00 State=UP AllowGroups=jfAccessToIridis5 PreemptMode=off
PartitionName=gpu nodes=indigo[51-60] ExclusiveUser=YES MaxCPUsPerNode=40 DefaultTime=02:00:00 MaxTime=60:00:00 QOS=gpu State=UP AllowGroups=jfAccessToIridis5 PreemptMode=off
PartitionName=gtx1080 nodes=pink[51-60] ExclusiveUser=YES MaxCPUsPerNode=54 DefaultTime=02:00:00 MaxTime=60:00:00 QOS=gtx1080 State=UP AllowGroups=jfAccessToIridis5 PreemptMode=off
PartitionName=lyceum nodes=pink[51-60] Shared=Yes MaxCPUsPerNode=54 DefaultTime=02:00:00 MaxTime=60:00:00 QOS=lyceum State=UP AllowAccounts=lyceum PreemptMode=off
PartitionName=serial nodes=red[460-464] Shared=Yes MaxCPUsPerNode=40 DefaultTime=02:00:00 MaxTime=60:00:00 QOS=serial SelectTypeParameters=CR_Core_Memory DefMemPerCPU=4300 State=UP AllowGroups=jfAccessToIridis5 PriorityJobFactor=10 PreemptMode=off
#PartitionName=relgroup nodes=red[465-470] ExclusiveUser=YES MaxCPUsPerNode=40 DefaultTime=02:00:00 MaxTime=60:00:00 QOS=relgroup State=UP AllowAccounts=relgroup Priority=10 PreemptMode=off
PartitionName=relgroup nodes=red[465-470] Shared=YES MaxCPUsPerNode=40 DefaultTime=02:00:00 MaxTime=60:00:00 QOS=relgroup State=UP AllowAccounts=relgroup SelectTypeParameters=CR_Core_Memory DefMemPerCPU=4300 Priority=10 PreemptMode=off
PartitionName=worldpop nodes=red[471-486] ExclusiveUser=YES MaxCPUsPerNode=40 DefaultTime=02:00:00 MaxTime=60:00:00 QOS=worldpop State=UP AllowAccounts=worldpop Priority=10 PreemptMode=off
PartitionName=hydrology nodes=red[487-490] ExclusiveUser=YES MaxCPUsPerNode=40 DefaultTime=02:00:00 MaxTime=60:00:00 QOS=hydrology State=UP AllowAccounts=hydrology Priority=10 PreemptMode=off

# Scavenger partition
# Disabled -- preemption only works if ExclusiveUser=NO
PartitionName=scavenger nodes=red[465-470] Shared=YES MaxCPUsPerNode=40 DefaultTime=00:15:00 MaxTime=02:00:00 QOS=scavenger State=UP AllowGroups=jfAccessToIridis5 SelectTypeParameters=CR_Core_Memory DefMemPerCPU=4300 PreemptMode=requeue

# High memory partition -- test only
PartitionName=highmem nodes=orange03 Shared=YES MaxCPUsPerNode=48 DefaultTime=02:00:00 MaxTime=60:00:00 State=UP AllowGroups=jfAccessToIridis5 SelectTypeParameters=CR_Core_Memory DefMemPerCPU=21200 PreemptMode=off

#PartitionName=lyceum nodes=pink[51-60] Shared=Yes MaxCPUsPerNode=54 DefaultTime=02:00:00 MaxTime=60:00:00 QOS=gtx1080 State=UP

# DCV
PartitionName=enginframe nodes=linuxvm[1-2] MaxTime=INFINITE Shared=YES  State=UP AllowGroups=jfAccessToIridis5 PreemptMode=off

#PartitionName=smp4.q Nodes=smp[4] SelectTypeParameters=CR_CORE State=UP Default=NO
#PartitionName=openmp7.q  nodes=cn7003  MaxTime=INFINITE  MaxCPUsPerNode=24 DefMemPerCPU=3900 Shared=YES  State=UP


