nf-core/configs: NYU HPC Configuration
All nf-core pipelines have been successfully configured for use on the HPC Cluster at New York University.
To use, run the pipeline with -profile nyu_hpc. This will download and launch the nyu_hpc.config which has been pre-configured with a setup suitable for the NYU HPC cluster.
Before running the pipeline you will need to load Nextflow using the environment module system on NYU HPC.
## See available nextflow versions
$ module avail nextflow
--- /share/apps/modulefiles ----
nextflow/25.04.7 nextflow/25.10.2
## Load desired nextflow
$ module load nextflow/25.10.2Config file
params {
config_profile_description = 'New York University HPC profile provided by nf-core/configs.'
config_profile_contact = 'HPC@nyu.edu'
config_profile_url = 'https://services.rt.nyu.edu'
// Resource limits
max_memory = 3000.GB
max_cpus = 128
max_time = 7.d
}
apptainer {
enabled = true
runOptions = "-B ${System.getenv('APPTAINER_BINDPATH') ?: '/scratch'}"
}
process {
resourceLimits = [
memory: params.max_memory,
cpus: params.max_cpus,
time: params.max_time
]
executor = 'slurm'
clusterOptions = {
"--account=${System.getenv('SLURM_JOB_ACCOUNT')} --export=NONE"
}
// Only retry for specific SLURM exit codes (e.g., OOM or Timeouts)
errorStrategy = { task.exitStatus in [143, 137, 104, 134, 139] ? 'retry' : 'finish' }
maxRetries = 3
cache = 'lenient' // Tolerates minor timestamp differences on shared filesystems
}
executor {
queueSize = 500
// Job submission throttling - prevents overwhelming scheduler
submitRateLimit = '3 sec' // Max 3 jobs/sec (180 jobs/min)
pollInterval = '10 sec' // Check running jobs every 10s (reduces I/O on shared FS)
queueStatInterval = '2 min' // Check queue status every 2min (reduces squeue load)
// SLURM optimization flags (version-dependent, gracefully ignored if unsupported)
queueGlobalStatus = true // Query jobs globally, not per-partition (23.01.0+)
onlyJobState = true // Use --only-job-state for faster queries (25.12.0+, requires SLURM 24.05+)
// Prevents false failures when file sync is delayed across storage nodes
exitReadTimeout = '10 min'
// Job submission retry with exponential backoff
retry {
maxAttempts = 5 // Try up to 5 times before giving up
delay = '5 sec' // Delay when retrying failed job submissions
maxDelay = '1 min' // Cap exponential backoff at 60sec
jitter = 0.25 // Add ±25% randomness to avoid retry storms
}
}