MPI4PY Python based job queue with caching

This post introduces a technique to spread a number of similar jobs into multiple workers using MPI, with the following characteristics:

  • There is a single scheduler and multiple workers.
  • The whole set of jobs can be interrupted and restarted, thanks to using a file-based cache.
  • The task to be spread among workers can be a Python closure or lambda function (which cannot be pickled!).

The caching and memoization is based on the pickle library, which we save into this tiny utility module.

import pickle
import os.path
def dump(filename, *data):
	with open(filename, "wb") as f:
		pickle.dump(data, f)

def load(filename):
	if os.path.exists(filename):
		with open(filename, "rb") as f:
			data = pickle.load(f)
		return data if len(data) > 1 else data[0]
		return None

The task scheduler is built on top of Python's mpi4py library and, more precisely, on the MPICommExecutor class. I do this because I tried to roll out my own class and it met with the busy-wait problem in MPICH—all processes, even those that had finished their work, maxed 100% CPU when waiting for other jobs to complete or for data to be communicated.

from tools import *
import platform
import os
from functools import reduce
import traceback

def default_task_runner(args):
	global _task_object
	return _task_object.execute(args)

class Tasker(object):

	def __init__(self, method='MPI'):
			self.comm = None
			if method == 'MPI':
				# MPI jobs, for most platforms, but only
				# if module is available
				from mpi4py import MPI
				self.comm = MPI.COMM_WORLD
				self.rank = self.comm.Get_rank()
				self.size = self.comm.Get_size()
				self.root = 0
		except Exception as e:
			print('Defaulting to serial jobs because mpi4py is not available.')
			self.comm = None
		if self.comm is None:
			# Backup system: serial jobs
			self.size = 1
			self.rank = self.root = 0
		self._prefix = f'Task {self.rank:3d}'
		self._function = None

	def isroot(self):
		return self.rank == self.root

	def __enter__(self):
		print(f'Entering tasker {self}')
		return self

	def __exit__(self, *args):
		return True

	def make_joblist(self, filename, args):
		dirname = filename + '.mpi'
		print(f'{self._prefix} Creating joblist for {filename} under {dirname}')
		if not os.path.exists(dirname):
		joblist = [(i, f'{dirname}/job{i:04d}.pkl', arg) for i, arg in enumerate(args)]
		return joblist, dirname

	def execute(self, arg):
		if self._function is None:
			raise Exception(f'{self._prefix} Function not installed')
		index, tmpfilename, *arguments = arg
		if os.path.exists(tmpfilename):
			print(f'{self._prefix} loads {tmpfilename}')
			data = load(tmpfilename)
			print(f'{self._prefix} executes {tmpfilename}')
			data = (index, arguments, self._function(*arguments))
			print(f'{self._prefix} saves {tmpfilename}')
			dump(tmpfilename, *data)
		print(f'{self._prefix} made {data}')
		return data

	def cleanup_files(self, dirname, joblist):
			for i, tmp, *_ in joblist:
				print(f'{self._prefix} removing file {tmp}')
		except Exception as e:
			print(f'{self._prefix} unable to remove directory {dirname}')

	def map(self, filename, fn, args):
		global _task_object
		self._function = fn
		if os.path.exists(filename):
			print(f'{self._prefix} File {filename} already exists. Returning it')
			return load(filename)
		elif self.size == 1:
			print(f'{self._prefix} Running jobs in serial mode')
			joblist, dirname = self.make_joblist(filename, args)
			output = [self.execute(arg) for arg in joblist]
			dump(filename, output)
			print(f'{self._prefix} Data saved into {filename}')
			self.cleanup_files(dirname, joblist)
			return output
				_task_object = self
				import mpi4py.futures
				with mpi4py.futures.MPICommExecutor(self.comm, self.root) as executor:
					print(f'{self._prefix} Executor is {executor}')
					if executor is not None:
						# Root process
						print(f'{self._prefix} Missing {filename}')
						joblist, dirname = self.make_joblist(filename, args)
						print(f'{self._prefix} got joblist with {len(joblist)} items')
						output = list(, joblist))
						print(f'{self._prefix} collected {len(output)} items')
						dump(filename, output)
						print(f'{self._prefix} Data saved into {filename}')
						self.cleanup_files(dirname, joblist)
			except Exception as e:
				print(f'{self._prefix} aborts due to exception {e}')
			_task_object = None
			return output

The following code shows how to use it. The same program is to be run on all processes. That program will create a `Tasker` object and use the `map()` method to distribut work among all processes.

from mp import Tasker
with Tasker() as tsk:
	print(f'task={tsk}, rank={tsk.rank}')
	data ='foo.pkl', lambda x: str(x), range(100))

The `Tasker` object will distribute 100 jobs among the `N-1` remaining processes, sending newer tasks to jobs that have already finished. Note also that the task to be done can be a closure and it may reference variables from the parent process. In order to run this code, I recommend using a non-buffering python instance. Assuming the code above was saved into the `` file, we can spread the 100 jobs into 3 processes plus 1 scheduler, as follows:

$ mpirun -n 4 python -u ./

The `Tasker` class will create a temporary directory with the `.mpi` extension. This directory will contain the memoized files. If your job crashes for some reason, such as exhausting the time allocated by the queue manager of your cluster, you can restart it and it will only run the tasks that were not completing. Once all tasks have been finished, the `*.mpi` directory will be deleted and pickle file with the right name, in this case `foo.pkl`, will be saved.