Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • skleff/multiprocessing-examples
  • lsaci/multiprocessing-examples
  • gsaurel/multiprocessing-examples
  • gepetto/multiprocessing-examples
4 results
Show changes
#!/bin/bash
#SBATCH --ntasks=1
#SBATCH --array=0-10
#SBATCH --mail-type=ALL
#SBATCH --output=output-%A-%a-%N.log
#SBATCH --error=output-%A-%a-%N.err
apptainer run --app minion pfc-3.sif
#!/usr/bin/env python3
"""Hard work here. Not efficient, but hard."""
from typing import Tuple
import pinocchio as pin # type: ignore
def is_prime(number: int) -> bool:
"""Check if a number is prime."""
return not any(number % i == 0 for i in range(2, number))
def find_prime(goal: int = 10, verbose: bool = False) -> Tuple[int, int]:
"""Find the goal-th prime number."""
found = 0
i = 1 # let's enumerate all numbers
while found < goal: # until we have enough prime ones.
i += 1
if is_prime(i):
found += 1
if verbose:
print(f"the prime number n°{found} is {i}")
n = pin.Quaternion(i, i, i, i).norm()
print(f"the norm of a quaternion with {i} as all members is {n}")
return i, n
if __name__ == "__main__":
print(find_prime(50, verbose=True))
# pfcalcul 3 - CUDA
pfc-3 but on CUDA
You'll need:
- understanding of pfc-3
- build the container, and send it to the pfc:
```bash
sudo apptainer build pfc-3g.sif pfc-3g.def
scp pfc-3g.sif $USER@pfcalcul.laas.fr:/pfcalcul/work/$USER/pfc-3g/
```
Then start a manager on the pfc frontend:
```bash
ssh $USER@pfcalcul.laas.fr
cd /pfcalcul/work/$USER/pfc-3g
apptainer run --app manager pfc-3g.sif
```
And run a boss wherever you want:
```
apptainer run --app boss pfc-3g.sif
```
Then, you should be good for `sbatch ./schedule.sh`
#!/usr/bin/env python3
import random
from manager import QueueClient
class Boss(QueueClient):
def run(self):
for _ in range(100):
task = random.randint(5, 1e4)
print("new task:", task)
self.queue.put(task)
if __name__ == "__main__":
Boss().run()
#!/usr/bin/env python3
import multiprocessing
import os
from multiprocessing.managers import BaseManager
PORT = 7481
KEY = b"AiZa5Uavcoh3PiajvaeTee5z" # keep it secret, keep it safe !
class QueueManager(BaseManager):
"""This Manager holds a Queue and waits for clients to use it."""
pass
class QueueClient:
"""Base class for users of the Queue."""
def __init__(self):
QueueManager.register("get_queue")
manager = QueueManager(
address=(os.environ.get("MANAGER_HOST", "localhost"), PORT), authkey=KEY
)
manager.connect()
self.queue = manager.get_queue()
if __name__ == "__main__":
queue = multiprocessing.Queue()
QueueManager.register("get_queue", callable=lambda: queue)
try:
QueueManager(address=("", PORT), authkey=KEY).get_server().serve_forever()
finally:
print()
print(f"exiting with approximately {queue.qsize()} items left in queue")
#!/usr/bin/env python3
"""
minion intended to be launched by slurm
"""
import argparse
import queue
import time
from manager import QueueClient
from work import find_prime
class Minion(QueueClient):
def __init__(self, n_tasks: int):
self.n_tasks = n_tasks
super().__init__()
def run(self):
for n in range(self.n_tasks):
try:
task = self.queue.get(block=False)
except queue.Empty:
print("All work is already done !")
break
start = time.perf_counter()
print(f"start work {n + 1}/{self.n_tasks} on task {task}...")
result, norm = find_prime(task)
perf = time.perf_counter() - start
print(f"The {task}-th prime number is {result}")
print(f"Its associated quaternion norm is {norm}")
print(f"found in {perf:.3f}s")
print("done !")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("n_tasks", type=int, default=1, nargs="?")
Minion(**vars(parser.parse_args())).run()
Bootstrap: docker
From: python:3.8-slim
%files
requirements.txt
%post
pip install --no-cache-dir -r requirements.txt
%environment
export MANAGER_HOST=pfcalcul.laas.fr
%appfiles minion
manager.py
work.py
minion.py
%appfiles manager
manager.py
%appfiles boss
manager.py
boss.py
%apprun minion
cd $SCIF_APPROOT
exec ./minion.py
%apprun manager
cd $SCIF_APPROOT
exec ./manager.py
%apprun boss
cd $SCIF_APPROOT
exec ./boss.py
pin
--extra-index-url https://download.pytorch.org/whl/cu113
torch
#!/bin/bash
#SBATCH --ntasks=1
#SBATCH --array=0-10
#SBATCH --mail-type=ALL
#SBATCH --output=output-%A-%a-%N.log
#SBATCH --error=output-%A-%a-%N.err
#SBATCH --gres=gpu:1
#SBATCH --partition=robgpu
#SBATCH --account=dept_rob
apptainer run --nv --app minion pfc-3g.sif
#!/usr/bin/env python3
"""Hard work here. Not efficient, but hard. On CUDA device."""
from typing import Tuple
import pinocchio as pin # type: ignore
import torch # type: ignore
def is_prime(number: int) -> bool:
"""Check if a number is prime."""
t = torch.arange(2, number, device="cuda")
return not bool((number % t == 0).any().cpu())
def find_prime(goal: int = 10, verbose: bool = False) -> Tuple[int, int]:
"""Find the goal-th prime number."""
found = 0
i = 1 # let's enumerate all numbers
while found < goal: # until we have enough prime ones.
i += 1
if is_prime(i):
found += 1
if verbose:
print(f"the prime number n°{found} is {i}")
n = pin.Quaternion(i, i, i, i).norm()
print(f"the norm of a quaternion with {i} as all members is {n}")
return i, n
if __name__ == "__main__":
print(find_prime(50, verbose=True))
[flake8]
max-line-length = 88