mirror of
https://gricad-gitlab.univ-grenoble-alpes.fr/tienyoun/presentations.git
synced 2026-09-26 21:41:08 +00:00
413 lines
9.6 KiB
Markdown
413 lines
9.6 KiB
Markdown
# Plan
|
|
|
|
- Qu'est ce que Celery
|
|
- Lancer une tâche
|
|
- Débugger Celery
|
|
- **La parallélisation et les sous-tâches**
|
|
- Worker pool
|
|
- Coordinator/Executor
|
|
- Les barres de progression dans Celery
|
|
|
|
|
|
|
|
## Worker pool
|
|
https://docs.celeryq.dev/en/main/userguide/workers.html#concurrency
|
|
- parallélisation dans un seul worker
|
|
- config: multiprocessing, threads ou coroutines
|
|
|
|
> Besoin de déclarer une limite max de sous-processus/threads
|
|
|
|
|
|
|
|
|
|
### Cas séquentiel
|
|
<div data-mermaid>
|
|
<script type="text/mermaid">
|
|
block
|
|
columns 1
|
|
block:macro_task
|
|
columns 7
|
|
space:3 macro_task_label["macro_task"] space:3
|
|
s(("start")) w1["work"] w2["work"] w3["work"] w4["work"] w5["work"] e(("end"))
|
|
s-->w1
|
|
w1-->w2
|
|
w2-->w3
|
|
w3-->w4
|
|
w4-->w5
|
|
w5-->e
|
|
end
|
|
classDef transparent fill:none,stroke:none;
|
|
class macro_task_label transparent
|
|
</script>
|
|
</div>
|
|
|
|
|
|
<div data-mermaid>
|
|
<script type="text/mermaid">
|
|
%%{init: { 'block': {'useMaxWidth':false, 'useWidth':1200}, 'theme':'neutral'} }%%
|
|
block
|
|
columns 1
|
|
block:macro_task
|
|
columns 1
|
|
worker_label["worker"]
|
|
block
|
|
columns 1
|
|
w1_label["thread1"]
|
|
w1["macro_task"]
|
|
end
|
|
block
|
|
columns 1
|
|
w2_label["thread2"]
|
|
w2["💤"]
|
|
end
|
|
block
|
|
columns 1
|
|
w3_label["thread3"]
|
|
w3["💤"]
|
|
end
|
|
end
|
|
classDef transparent fill:none,stroke:none;
|
|
classDef task fill:#ffffded9;
|
|
class worker_label,w1_label,w2_label,w2,w3_label,w3 transparent
|
|
class w1 task
|
|
</script>
|
|
</div>
|
|
|
|
<div data-mermaid>
|
|
<script type="text/mermaid">
|
|
%%{init: { 'theme':'forest'} }%%
|
|
block
|
|
columns 1
|
|
block:queue
|
|
columns 1
|
|
label["queue"]
|
|
space:5
|
|
end
|
|
classDef transparent fill:none,stroke:none;
|
|
classDef task fill:#ffffded9;
|
|
class label,w1_label,w2_label,w2,w3_label,w3 transparent
|
|
class w1 task
|
|
</script>
|
|
</div>
|
|
|
|
|
|
|
|
### Cas parallèle
|
|
|
|
<div data-mermaid>
|
|
<script type="text/mermaid">
|
|
block
|
|
columns 1
|
|
block:macro_task
|
|
columns 3
|
|
macro_task_label["macro_task"]:3
|
|
space
|
|
block:w1
|
|
columns 1
|
|
w1_label["micro_task"]
|
|
w1_["work"]
|
|
end
|
|
space
|
|
s(("start"))
|
|
block:w2
|
|
columns 1
|
|
w2_label["micro_task"]
|
|
w2_["work"]
|
|
end
|
|
e(("end"))
|
|
space
|
|
block:w3
|
|
columns 1
|
|
w3_label["micro_task"]
|
|
w3_["work"]
|
|
end
|
|
end
|
|
s-->w1
|
|
s-->w2
|
|
s-->w3
|
|
w1-->e
|
|
w2-->e
|
|
w3-->e
|
|
classDef transparent fill:none,stroke:none;
|
|
classDef lg fill:lightgray,stroke:none;
|
|
class macro_task_label,w1_label,w2_label,w3_label,w4_label,w5_label,s_ transparent
|
|
class w1,w2,w3,w4,w5 lg
|
|
</script>
|
|
</div>
|
|
|
|
<div data-mermaid >
|
|
<script type="text/mermaid">
|
|
---
|
|
config:
|
|
theme: neutral
|
|
---
|
|
block
|
|
columns 1
|
|
block:macro_task
|
|
columns 1
|
|
worker_label["worker"]
|
|
block
|
|
columns 1
|
|
w1_label["thread1"]
|
|
w1["macro_task"]
|
|
end
|
|
block
|
|
columns 1
|
|
w2_label["thread2"]
|
|
w2["micro_task"]
|
|
end
|
|
block
|
|
columns 1
|
|
w3_label["thread3"]
|
|
w3["micro_task"]
|
|
end
|
|
end
|
|
classDef transparent fill:none,stroke:none;
|
|
classDef task fill:#ffffded9;
|
|
class worker_label,w1_label,w2_label,w3_label transparent
|
|
class w1 task
|
|
</script>
|
|
</div>
|
|
<div data-mermaid>
|
|
<script type="text/mermaid">
|
|
---
|
|
config:
|
|
theme: forest
|
|
---
|
|
block
|
|
columns 1
|
|
block:queue
|
|
columns 1
|
|
label["queue"]
|
|
w1["micro_task"]
|
|
end
|
|
classDef transparent fill:none,stroke:none;
|
|
classDef task fill:#eee,stroke:#999;
|
|
class label transparent
|
|
class w1 task
|
|
</script>
|
|
</div>
|
|
|
|
|
|
|
|
### Cas parallèle
|
|
|
|
<div data-mermaid>
|
|
<script type="text/mermaid">
|
|
block
|
|
columns 1
|
|
block:macro_task
|
|
columns 3
|
|
macro_task_label["macro_task"]:3
|
|
space
|
|
block:w1
|
|
columns 1
|
|
w1_label["micro_task"]
|
|
w1_["work"]
|
|
end
|
|
space
|
|
s(("start"))
|
|
block:w2
|
|
columns 1
|
|
w2_label["micro_task"]
|
|
w2_["work"]
|
|
end
|
|
e(("end"))
|
|
space
|
|
block:w3
|
|
columns 1
|
|
w3_label["micro_task"]
|
|
w3_["work"]
|
|
end
|
|
end
|
|
s-->w1
|
|
s-->w2
|
|
s-->w3
|
|
w1-->e
|
|
w2-->e
|
|
w3-->e
|
|
classDef transparent fill:none,stroke:none;
|
|
classDef lg fill:lightgray,stroke:none;
|
|
class macro_task_label,w1_label,w2_label,w3_label,w4_label,w5_label,s_ transparent
|
|
class w1,w2,w3,w4,w5 lg
|
|
</script>
|
|
</div>
|
|
<div data-mermaid>
|
|
<script type="text/mermaid">
|
|
---
|
|
config:
|
|
theme: neutral
|
|
---
|
|
block
|
|
columns 1
|
|
block:macro_task
|
|
columns 1
|
|
worker_label["worker"]
|
|
block
|
|
columns 1
|
|
w1_label["thread1"]
|
|
w1["macro_task"]
|
|
end
|
|
block
|
|
columns 1
|
|
w2_label["thread2"]
|
|
w2["macro_task"]
|
|
end
|
|
block
|
|
columns 1
|
|
w3_label["thread3"]
|
|
w3["macro_task"]
|
|
end
|
|
end
|
|
classDef transparent fill:none,stroke:none;
|
|
classDef task fill:#ffffded9;
|
|
class worker_label,w1_label,w2_label,w3_label transparent
|
|
class w1,w2,w3 task
|
|
</script>
|
|
</div>
|
|
<div data-mermaid>
|
|
<script type="text/mermaid">
|
|
---
|
|
config:
|
|
theme: forest
|
|
---
|
|
block
|
|
columns 1
|
|
block:queue
|
|
columns 1
|
|
label["queue"]
|
|
w1["micro_task"]
|
|
w2["micro_task"]
|
|
w3["micro_task"]
|
|
w4["micro_task"]
|
|
w5["micro_task"]
|
|
w6["micro_task"]
|
|
w7["micro_task"]
|
|
w8["micro_task"]
|
|
w9["micro_task"]
|
|
end
|
|
classDef transparent fill:none,stroke:none;
|
|
classDef task fill:#eee,stroke:#999;
|
|
class label transparent
|
|
class w1,w2,w3,w4,w5,w6,w7,w8,w9 task
|
|
</script>
|
|
</div>
|
|
|
|
|
|
|
|
### Coordinator/Executor
|
|
<div data-mermaid>
|
|
<script type="text/mermaid">
|
|
---
|
|
config:
|
|
theme: neutral
|
|
---
|
|
block
|
|
columns 1
|
|
block:macro_task
|
|
columns 1
|
|
worker_label["coordinator worker"]
|
|
block
|
|
columns 1
|
|
w1_label["thread1"]
|
|
w1["macro_task"]
|
|
end
|
|
block
|
|
columns 1
|
|
w2_label["thread2"]
|
|
w2["macro_task"]
|
|
end
|
|
block
|
|
columns 1
|
|
w3_label["thread3"]
|
|
w3["macro_task"]
|
|
end
|
|
end
|
|
classDef transparent fill:none,stroke:none;
|
|
classDef task fill:#ffffded9;
|
|
class worker_label,w1_label,w2_label,w3_label transparent
|
|
class w1,w2,w3 task
|
|
</script>
|
|
</div>
|
|
<div data-mermaid>
|
|
<script type="text/mermaid">
|
|
---
|
|
config:
|
|
theme: forest
|
|
---
|
|
block
|
|
columns 1
|
|
block:queue
|
|
columns 1
|
|
label["coordinator queue"]
|
|
w1["macro_task"]
|
|
w2["macro_task"]
|
|
w3["macro_task"]
|
|
space:5
|
|
end
|
|
classDef transparent fill:none,stroke:none;
|
|
classDef task fill:#ffffded9,stroke:#999;
|
|
class label transparent
|
|
class w1,w2,w3,w4,w5,w6,w7,w8,w9 task
|
|
</script>
|
|
</div>
|
|
|
|
<div data-mermaid>
|
|
<script type="text/mermaid">
|
|
---
|
|
config:
|
|
theme: neutral
|
|
---
|
|
block
|
|
columns 1
|
|
block:macro_task
|
|
columns 1
|
|
worker_label["executor worker"]
|
|
block
|
|
columns 1
|
|
w1_label["thread1"]
|
|
w1["micro_task"]
|
|
end
|
|
block
|
|
columns 1
|
|
w2_label["thread2"]
|
|
w2["micro_task"]
|
|
end
|
|
block
|
|
columns 1
|
|
w3_label["thread3"]
|
|
w3["micro_task"]
|
|
end
|
|
end
|
|
classDef transparent fill:none,stroke:none;
|
|
classDef task fill:#eee,stroke:#999;
|
|
class worker_label,w1_label,w2_label,w3_label transparent
|
|
class w1,w2,w3 task
|
|
</script>
|
|
</div>
|
|
<div data-mermaid>
|
|
<script type="text/mermaid">
|
|
---
|
|
config:
|
|
theme: forest
|
|
---
|
|
block
|
|
columns 1
|
|
block:queue
|
|
columns 1
|
|
label["executor queue"]
|
|
w1["micro_task"]
|
|
w2["micro_task"]
|
|
w3["micro_task"]
|
|
w4["micro_task"]
|
|
w5["micro_task"]
|
|
w6["micro_task"]
|
|
space:2
|
|
end
|
|
classDef transparent fill:none,stroke:none;
|
|
classDef task fill:#eee,stroke:#999;
|
|
class label transparent
|
|
class w1,w2,w3,w4,w5,w6,w7,w8,w9 task
|
|
</script>
|
|
</div>
|