cp pfc-1 pfc-2

pfc-2/README.md

+# pfcalcul 1
+
+Implementation of example-1 for HPC @ LAAS
+
+You'll need:
+
+- understanding of example-1
+- an account on pfcalcul (doc is available at https://pfcalcul.laas.fr/)
+- a copy of this example in `/pfcalcul/work/$USER/pfc-1`
+- start the manager `./manager.py`. This might be on the `pfcalcul.laas.fr` frontend, but the real requirements are:
+    - must be reachable by network from the pfcalcul nodes
+    - must stay available for the whole job time
+- populate the Queue with tasks from a master (eg. on your personnal computer): `HOST=pfcalcul.laas.fr ./boss.py`
+- ask the pfcalcul to schedule enough minions : `sbatch ./schedule.sh`
+
+After that, you should get a job number. And you'll get mails for everything.
+
+You can show detailed informations about your job with `scontrol show jobid <job_number>`
+
+## Nota Bene
+
+This example start 11 slurm jobs (0 to 10), each one using a single CPU thread, and running a Minion configure to
+handle 10 of the tasks defined by the Boss. The Boss puts only 100 tasks in the Queue, so some minions will stop early.

pfc-2/boss.py

+#!/usr/bin/env python3
+
+import random
+import time
+
+from manager import QueueClient
+
+
+class Boss(QueueClient):
+    def run(self):
+        for _ in range(100):
+            task = random.randint(5, 1e4)
+            print('new task:', task)
+            self.queue.put(task)
+
+
+if __name__ == '__main__':
+    Boss().run()

pfc-2/manager.py

+#!/usr/bin/env python3
+
+import multiprocessing
+import os
+from multiprocessing.managers import BaseManager
+
+PORT = 7481
+KEY = b'AiZa5Uavcoh3PiajvaeTee5z'  # keep it secret, keep it safe !
+
+
+class QueueManager(BaseManager):
+    """This Manager holds a Queue and waits for clients to use it."""
+    pass
+
+
+class QueueClient:
+    """Base class for users of the Queue."""
+    def __init__(self):
+        QueueManager.register('get_queue')
+        manager = QueueManager(address=(os.environ.get('HOST', 'localhost'), PORT), authkey=KEY)
+        manager.connect()
+        self.queue = manager.get_queue()
+
+
+if __name__ == '__main__':
+    queue = multiprocessing.Queue()
+    QueueManager.register('get_queue', callable=lambda: queue)
+    QueueManager(address=('', PORT), authkey=KEY).get_server().serve_forever()

pfc-2/minion.py

+#!/usr/bin/env python3
+"""
+minion intended to be launched by slurm
+"""
+
+import argparse
+import queue
+import random
+import time
+
+from manager import QueueClient
+from work import find_prime
+
+
+class Minion(QueueClient):
+    def __init__(self, n_tasks: int):
+        self.n_tasks = n_tasks
+        super().__init__()
+
+    def run(self):
+        for n in range(self.n_tasks):
+            try:
+                task = self.queue.get(block=False)
+            except queue.Empty:
+                print('All work is already done !')
+                break
+            start = time.perf_counter()
+            print(f"start work {n}/{self.n_tasks} on task {task}...")
+            result = find_prime(task)
+            perf = time.perf_counter() - start
+            print(f"The {task}-th prime number is {result} (found in {perf:.3f}s)")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("n_tasks", type=int, default=1, nargs='?')
+
+    Minion(**vars(parser.parse_args())).run()

pfc-2/schedule.sh

+#!/bin/bash
+
+#SBATCH --ntasks=1
+#SBATCH --array=0-10
+#SBATCH --mail-type=ALL
+#SBATCH --output=output-%A-%a-%N.log
+#SBATCH --error=output-%A-%a-%N.err
+
+HOST=pfcalcul.laas.fr ./minion.py 10

pfc-2/work.py

+#!/usr/bin/env python3
+"""Hard work here. Not efficient, but hard."""
+
+
+def is_prime(number: int) -> bool:
+    """Check if a number is prime."""
+    return not any(number % i == 0 for i in range(2, number))
+
+
+def find_prime(goal: int = 10, verbose: bool = False) -> int:
+    """Find the goal-th prime number."""
+    found = 0
+    i = 1  # let's enumerate all numbers
+    while found < goal:  # until we have enough prime ones.
+        i += 1
+        if is_prime(i):
+            found += 1
+            if verbose:
+                print(f'the prime number n°{found} is {i}')
+    return i
+
+
+if __name__ == '__main__':
+    print(find_prime(50, verbose=True))