# Plan

- Qu'est ce que Celery
- Lancer une tâche
- Débugger Celery
- **La parallélisation et les sous-tâches**
    - Worker pool
    - Coordinator/Executor
- Les barres de progression dans Celery



## Worker pool
https://docs.celeryq.dev/en/main/userguide/workers.html#concurrency
- parallélisation dans un seul worker
- config: multiprocessing, threads ou coroutines

> Besoin de déclarer une limite max de sous-processus/threads




### Cas séquentiel
<div data-mermaid>
    <script type="text/mermaid">
    block
        columns 1
        block:macro_task
            columns 7
            space:3 macro_task_label["macro_task"] space:3
            s(("start")) w1["work"] w2["work"] w3["work"] w4["work"] w5["work"] e(("end")) 
            s-->w1
            w1-->w2
            w2-->w3
            w3-->w4
            w4-->w5
            w5-->e
        end
        classDef transparent fill:none,stroke:none;
        class macro_task_label transparent
    </script>
</div>


<div data-mermaid>
    <script type="text/mermaid">
    %%{init: { 'block': {'useMaxWidth':false, 'useWidth':1200}, 'theme':'neutral'} }%%
    block
        columns 1
        block:macro_task
            columns 1
            worker_label["worker"]
            block
                columns 1
                w1_label["thread1"]
                w1["macro_task"]
            end
            block
                columns 1
                w2_label["thread2"]
                w2["💤"]
            end
            block
                columns 1
                w3_label["thread3"]
                w3["💤"]
            end
        end
        classDef transparent fill:none,stroke:none;
        classDef task fill:#ffffded9;
        class worker_label,w1_label,w2_label,w2,w3_label,w3 transparent
        class w1 task
    </script>
</div>

<div data-mermaid>
    <script type="text/mermaid">
    %%{init: { 'theme':'forest'} }%%
    block
        columns 1
        block:queue
            columns 1
            label["queue"]
            space:5
        end
        classDef transparent fill:none,stroke:none;
        classDef task fill:#ffffded9;
        class label,w1_label,w2_label,w2,w3_label,w3 transparent
        class w1 task
    </script>
</div>



### Cas parallèle

<div data-mermaid>
    <script type="text/mermaid">
    block
        columns 1
        block:macro_task
            columns 3
            macro_task_label["macro_task"]:3
            space
            block:w1
                columns 1
                w1_label["micro_task"]
                w1_["work"]
            end
            space
            s(("start"))
            block:w2
                columns 1
                w2_label["micro_task"]
                w2_["work"]
            end
            e(("end")) 
            space
            block:w3
                columns 1
                w3_label["micro_task"]
                w3_["work"]
            end
        end
        s-->w1
        s-->w2
        s-->w3
        w1-->e
        w2-->e
        w3-->e
        classDef transparent fill:none,stroke:none;
        classDef lg fill:lightgray,stroke:none;
        class macro_task_label,w1_label,w2_label,w3_label,w4_label,w5_label,s_ transparent
        class w1,w2,w3,w4,w5 lg
    </script>
</div>

<div data-mermaid >
    <script type="text/mermaid">
    ---
    config:
        theme: neutral
    ---
    block
        columns 1
        block:macro_task
            columns 1
            worker_label["worker"]
            block
                columns 1
                w1_label["thread1"]
                w1["macro_task"]
            end
            block
                columns 1
                w2_label["thread2"]
                w2["micro_task"]
            end
            block
                columns 1
                w3_label["thread3"]
                w3["micro_task"]
            end
        end
        classDef transparent fill:none,stroke:none;
        classDef task fill:#ffffded9;
        class worker_label,w1_label,w2_label,w3_label transparent
        class w1 task
    </script>
</div>
<div data-mermaid>
    <script type="text/mermaid">
    ---
    config:
        theme: forest
    ---
    block
        columns 1
        block:queue
            columns 1
            label["queue"]
            w1["micro_task"]
        end
        classDef transparent fill:none,stroke:none;
        classDef task fill:#eee,stroke:#999;
        class label transparent
        class w1 task
    </script>
</div>



### Cas parallèle

<div data-mermaid>
    <script type="text/mermaid">
    block
        columns 1
        block:macro_task
            columns 3
            macro_task_label["macro_task"]:3
            space
            block:w1
                columns 1
                w1_label["micro_task"]
                w1_["work"]
            end
            space
            s(("start"))
            block:w2
                columns 1
                w2_label["micro_task"]
                w2_["work"]
            end
            e(("end")) 
            space
            block:w3
                columns 1
                w3_label["micro_task"]
                w3_["work"]
            end
        end
        s-->w1
        s-->w2
        s-->w3
        w1-->e
        w2-->e
        w3-->e
        classDef transparent fill:none,stroke:none;
        classDef lg fill:lightgray,stroke:none;
        class macro_task_label,w1_label,w2_label,w3_label,w4_label,w5_label,s_ transparent
        class w1,w2,w3,w4,w5 lg
    </script>
</div>
<div data-mermaid>
    <script type="text/mermaid">
    ---
    config:
        theme: neutral
    ---
    block
        columns 1
        block:macro_task
            columns 1
            worker_label["worker"]
            block
                columns 1
                w1_label["thread1"]
                w1["macro_task"]
            end
            block
                columns 1
                w2_label["thread2"]
                w2["macro_task"]
            end
            block
                columns 1
                w3_label["thread3"]
                w3["macro_task"]
            end
        end
        classDef transparent fill:none,stroke:none;
        classDef task fill:#ffffded9;
        class worker_label,w1_label,w2_label,w3_label transparent
        class w1,w2,w3 task
    </script>
</div>
<div data-mermaid>
    <script type="text/mermaid">
    ---
    config:
        theme: forest
    ---
    block
        columns 1
        block:queue
            columns 1
            label["queue"]
            w1["micro_task"]
            w2["micro_task"]
            w3["micro_task"]
            w4["micro_task"]
            w5["micro_task"]
            w6["micro_task"]
            w7["micro_task"]
            w8["micro_task"]
            w9["micro_task"]
        end
        classDef transparent fill:none,stroke:none;
        classDef task fill:#eee,stroke:#999;
        class label transparent
        class w1,w2,w3,w4,w5,w6,w7,w8,w9 task
    </script>
</div>



### Coordinator/Executor
<div data-mermaid>
    <script type="text/mermaid">
    ---
    config:
        theme: neutral
    ---
    block
        columns 1
        block:macro_task
            columns 1
            worker_label["coordinator worker"]
            block
                columns 1
                w1_label["thread1"]
                w1["macro_task"]
            end
            block
                columns 1
                w2_label["thread2"]
                w2["macro_task"]
            end
            block
                columns 1
                w3_label["thread3"]
                w3["macro_task"]
            end
        end
        classDef transparent fill:none,stroke:none;
        classDef task fill:#ffffded9;
        class worker_label,w1_label,w2_label,w3_label transparent
        class w1,w2,w3 task
    </script>
</div>
<div data-mermaid>
    <script type="text/mermaid">
    ---
    config:
        theme: forest
    ---
    block
        columns 1
        block:queue
            columns 1
            label["coordinator queue"]
            w1["macro_task"]
            w2["macro_task"]
            w3["macro_task"]
            space:5
        end
        classDef transparent fill:none,stroke:none;
        classDef task fill:#ffffded9,stroke:#999;
        class label transparent
        class w1,w2,w3,w4,w5,w6,w7,w8,w9 task
    </script>
</div>

<div data-mermaid>
    <script type="text/mermaid">
    ---
    config:
        theme: neutral
    ---
    block
        columns 1
        block:macro_task
            columns 1
            worker_label["executor worker"]
            block
                columns 1
                w1_label["thread1"]
                w1["micro_task"]
            end
            block
                columns 1
                w2_label["thread2"]
                w2["micro_task"]
            end
            block
                columns 1
                w3_label["thread3"]
                w3["micro_task"]
            end
        end
        classDef transparent fill:none,stroke:none;
        classDef task fill:#eee,stroke:#999;
        class worker_label,w1_label,w2_label,w3_label transparent
        class w1,w2,w3 task
    </script>
</div>
<div data-mermaid>
    <script type="text/mermaid">
    ---
    config:
        theme: forest
    ---
    block
        columns 1
        block:queue
            columns 1
            label["executor queue"]
            w1["micro_task"]
            w2["micro_task"]
            w3["micro_task"]
            w4["micro_task"]
            w5["micro_task"]
            w6["micro_task"]
            space:2
        end
        classDef transparent fill:none,stroke:none;
        classDef task fill:#eee,stroke:#999;
        class label transparent
        class w1,w2,w3,w4,w5,w6,w7,w8,w9 task
    </script>
</div>