from joblib import Parallel, delayed import multiprocessing import time import os import numpy # define some data inputs = range(100) # function to execute in parallel def myFun(i): # Perform some computations for x in range (0, 10000): y = numpy.mean( numpy.random.normal(0, 1, 10000)) return i * i # detect how many cores machine has num_cores = multiprocessing.cpu_count() print ("This machine has " + str(num_cores) + " cores") # get the number of cores that was requested by the submission script num_cores_requested = int( os.getenv('NSLOTS') ) print ("The number of slots requested by the batch script is " + str(num_cores_requested) ) # use only as many cores as you have requested results = Parallel( n_jobs = num_cores_requested )( delayed( myFun )(i) for i in inputs) print ("The result array is " + ', '.join( str(x) for x in results))