mirror of
https://gricad-gitlab.univ-grenoble-alpes.fr/tienyoun/presentations.git
synced 2026-09-26 21:41:08 +00:00
Celery
This commit is contained in:
Nathan Tien You
Binary file not shown.
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 93 KiB |
Binary file not shown.
@@ -0,0 +1,404 @@
|
||||
/*
|
||||
Plugin for embedding mermaid.js diagrams in reveal.js presentations
|
||||
GitHub: https://github.com/dainiak/revealjs-plugins/
|
||||
|
||||
Author: Alex Dainiak
|
||||
Web: www.dainiak.com
|
||||
Email: dainiak@gmail.com
|
||||
|
||||
Processing rules can be added to post-process the diagram SVG DOM subtree.
|
||||
One can set the id, set or remove classes, or set attributes on the SVG elements. In any combinations:
|
||||
%%% someCssSelector -> #cssIdToSet .css-class-to-set !.class-to-remove [some-property-to-set=newValue]
|
||||
If the `->` arrow is used then the modifications are applied to the element itself.
|
||||
If the `^->` arrow is used then the element is wrapped with a g element and modifications are applied to this g.
|
||||
If the `_->` arrow is used then a single g is added as the only new child of the element and the modifications are applied to this g. All the former children are moved inside this g.
|
||||
You can use [n] right after the selector to target a specific element in the list of matching elements. Works even when `:nth-child` does not.
|
||||
Examples:
|
||||
%% Apply .fragment class to the first of the mermaid nodes:
|
||||
%%% g.node[1] -> .fragment
|
||||
|
||||
%% Make the "Start" node a fragment step and red
|
||||
%%% rect[id*="A"] _-> .fragment .fade-up [data-fragment-index=1]
|
||||
|
||||
%% Remove default class and add new one
|
||||
%%% #D -> !.node .end-state
|
||||
*/
|
||||
|
||||
const RevealMermaid = {
|
||||
id: 'mermaid',
|
||||
init: async (reveal) => {
|
||||
const katexVersion = '0.16.27';
|
||||
const mermaidVersion = '11.14.0';
|
||||
let options = reveal.getConfig().mermaid || {};
|
||||
options = {
|
||||
mathInLabels: options.mathInLabels !== false,
|
||||
urls: options.urls || {
|
||||
mermaid: options.urls && options.urls.mermaid || `https://cdn.jsdelivr.net/npm/mermaid@${mermaidVersion}/dist/mermaid.min.js`,
|
||||
katex: options.urls && options.urls.katex || `https://cdn.jsdelivr.net/npm/katex@${katexVersion}/dist/katex.min.js`,
|
||||
katexCss: options.urls && options.urls.katexCss || `https://cdn.jsdelivr.net/npm/katex@${katexVersion}/dist/katex.min.css`,
|
||||
},
|
||||
selectors: {
|
||||
container: options.selectors && options.selectors.container || '[data-mermaid]',
|
||||
script: options.selectors && options.selectors.script || 'script[type="text/mermaid"]',
|
||||
},
|
||||
overflowVisible: options.overflowVisible === undefined ? true : options.overflowVisible,
|
||||
mermaidInit: options.mermaidInit || {
|
||||
startOnLoad: false,
|
||||
theme: 'auto',
|
||||
suppressErrorRendering: true
|
||||
},
|
||||
iconPacks: options.iconPacks || [],
|
||||
css: {
|
||||
enabled: options?.css?.enabled !== false,
|
||||
cssIndices: options?.css?.cssIndices !== false,
|
||||
indexClassPrefix: options?.css?.indexClassPrefix || 'fragidx-',
|
||||
debug: options?.css?.debug || false
|
||||
}
|
||||
};
|
||||
|
||||
let scriptsToLoad = [
|
||||
{
|
||||
url: options.urls.mermaid,
|
||||
condition:
|
||||
!window.mermaid
|
||||
&& !document.querySelector(`script[src="${options.mermaidUrl}"]`)
|
||||
}, {
|
||||
url: options.urls.katex,
|
||||
condition:
|
||||
options.mathInLabels
|
||||
&& !window.katex
|
||||
&& !document.querySelector(`script[src="${options.katexUrl}"]`)
|
||||
},{
|
||||
url: options.urls.katexCss,
|
||||
type: 'text/css',
|
||||
condition:
|
||||
!window.vegaEmbed
|
||||
&& !window.katex
|
||||
}
|
||||
];
|
||||
|
||||
function loadScript(params) {
|
||||
return new Promise((resolve) => {
|
||||
if(params.condition !== undefined
|
||||
&& !(params.condition === true || typeof params.condition == 'function' && params.condition.call())) {
|
||||
return resolve();
|
||||
}
|
||||
|
||||
if( params.type === undefined )
|
||||
params.type = (params.url && params.url.match(/\.css[^.]*$/)) ? 'text/css' : 'text/javascript';
|
||||
|
||||
let element;
|
||||
|
||||
if( params.type === 'text/css' ){
|
||||
if( params.content ){
|
||||
element = document.createElement('style');
|
||||
element.textContent = params.content;
|
||||
}
|
||||
else {
|
||||
element = document.createElement('link');
|
||||
element.rel = 'stylesheet';
|
||||
element.type = 'text/css';
|
||||
element.href = params.url;
|
||||
}
|
||||
}
|
||||
else {
|
||||
element = document.createElement('script');
|
||||
element.type = params.type || 'text/javascript';
|
||||
if( params.content )
|
||||
element.textContent = params.content;
|
||||
else
|
||||
element.src = params.url;
|
||||
}
|
||||
|
||||
if(params.content){
|
||||
document.querySelector('head').appendChild(element);
|
||||
resolve();
|
||||
}
|
||||
else {
|
||||
element.onload = resolve;
|
||||
document.querySelector( 'head' ).appendChild(element);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
if(!reveal.getSlidesElement().querySelector(options.selectors.container))
|
||||
return;
|
||||
|
||||
|
||||
// Load mermaid, katex, and katex CSS in parallel (they are independent)
|
||||
await Promise.all(scriptsToLoad.map(s => loadScript(s)));
|
||||
|
||||
if((options.mermaidInit.theme || 'auto') === 'auto') {
|
||||
options.mermaidInit.theme = 'default';
|
||||
if(document.querySelector(
|
||||
'[href*="black.css"],[href*="league.css"],[href*="night.css"],[href*="moon.css"],[href*="dracula.css"],[href*="blood.css"]'
|
||||
)) {
|
||||
options.mermaidInit.theme = 'dark';
|
||||
}
|
||||
}
|
||||
|
||||
window.mermaid.initialize(options.mermaidInit);
|
||||
|
||||
if(options.iconPacks) {
|
||||
for(let packName in options.iconPacks) {
|
||||
window.mermaid.registerIconPacks([
|
||||
{
|
||||
name: packName,
|
||||
loader: () =>
|
||||
fetch(options.iconPacks[packName]).then((res) => res.json()),
|
||||
},
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function dedentAndTrim(str) {
|
||||
const TAB_WIDTH = 4;
|
||||
const lines = str.split(/\r?\n/);
|
||||
let minIndent = Infinity;
|
||||
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
let width = 0;
|
||||
for (const char of line) {
|
||||
if (char === ' ') width++;
|
||||
else if (char === '\t') width += TAB_WIDTH;
|
||||
else break;
|
||||
}
|
||||
if (width < minIndent) minIndent = width;
|
||||
}
|
||||
|
||||
if (minIndent === Infinity) minIndent = 0;
|
||||
|
||||
return lines.map(line => {
|
||||
let width = 0, i = 0;
|
||||
for (; i < line.length; i++) {
|
||||
if (width >= minIndent) break;
|
||||
const char = line[i];
|
||||
if (char === ' ') width++;
|
||||
else if (char === '\t') width += TAB_WIDTH;
|
||||
else break;
|
||||
}
|
||||
return ' '.repeat(Math.max(0, width - minIndent)) + line.slice(i);
|
||||
}).join('\n').trim();
|
||||
}
|
||||
|
||||
const mermaidContainers = Array.from(reveal.getSlidesElement().querySelectorAll(options.selectors.container));
|
||||
|
||||
const renderPromises = mermaidContainers.map(async (mermaidContainer) => {
|
||||
const parent = mermaidContainer.parentNode;
|
||||
const newDiv = document.createElement('div');
|
||||
parent.insertBefore(newDiv, mermaidContainer);
|
||||
|
||||
const mermaidScript = mermaidContainer.querySelector("script");
|
||||
if(mermaidScript) {
|
||||
mermaidScript.textContent = dedentAndTrim(mermaidScript.textContent);
|
||||
}
|
||||
|
||||
if(!mermaidContainer.id)
|
||||
mermaidContainer.id = `mermaid-${Math.floor(Math.random() * 1000000)}`;
|
||||
|
||||
let graphDefinition;
|
||||
const mermaidAttrValue = mermaidContainer.getAttribute('data-mermaid');
|
||||
if(mermaidAttrValue && mermaidAttrValue.trim() !== '') {
|
||||
try {
|
||||
const response = await fetch(mermaidAttrValue.trim());
|
||||
if(!response.ok)
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
graphDefinition = await response.text();
|
||||
} catch(error) {
|
||||
console.warn(`Mermaid Plugin: Failed to fetch diagram from "${mermaidAttrValue}":`, error);
|
||||
mermaidContainer.remove();
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
const scriptEl = mermaidContainer.querySelector(options.selectors.script);
|
||||
if(!scriptEl) {
|
||||
console.warn('Mermaid Plugin: No script element or data-mermaid path found in container', mermaidContainer);
|
||||
mermaidContainer.remove();
|
||||
return;
|
||||
}
|
||||
graphDefinition = scriptEl.innerHTML;
|
||||
}
|
||||
let renderRules = [];
|
||||
|
||||
// 1. Extract Custom CSS Logic
|
||||
// Regex matches: selector + (one of ->, ^->, _->) + assignment
|
||||
const ruleRegex = /^\s*%%%\s+(.+?)\s+([\^_]?->)\s+(.+?)\s*$/gm;
|
||||
|
||||
graphDefinition = graphDefinition.replace(ruleRegex, (match, selector, arrow, assignment) => {
|
||||
renderRules.push({
|
||||
selector,
|
||||
assignment,
|
||||
type: arrow
|
||||
});
|
||||
return match;
|
||||
});
|
||||
|
||||
// 2. Handle Math in Labels
|
||||
if(options.mathInLabels) {
|
||||
graphDefinition = graphDefinition.replace(/\\([(\[]).*?\\([)\]])/g, (s) => {
|
||||
let output = window.katex.renderToString(s.substring(2, s.length - 2), {
|
||||
output: 'html',
|
||||
displayMode: s[1] === '['
|
||||
});
|
||||
return output.replaceAll('"', "'");
|
||||
});
|
||||
}
|
||||
|
||||
// 3. Render Mermaid
|
||||
try {
|
||||
await window.mermaid.parse(graphDefinition, {suppressErrors: false});
|
||||
} catch (error) {
|
||||
console.warn(`Mermaid diagram failed to parse:\n\n${graphDefinition}\n\nError: `, error);
|
||||
mermaidContainer.remove();
|
||||
return;
|
||||
}
|
||||
|
||||
let svg;
|
||||
try {
|
||||
({ svg } = await window.mermaid.render(mermaidContainer.id, graphDefinition));
|
||||
} catch (error) {
|
||||
console.warn(`Mermaid diagram failed to render:\n\n${graphDefinition}\n\nError: `, error);
|
||||
mermaidContainer.remove();
|
||||
return;
|
||||
}
|
||||
newDiv.outerHTML = svg;
|
||||
const svgElement = parent.querySelector(`#${mermaidContainer.id}`);
|
||||
|
||||
// Copy classes and styles from container
|
||||
svgElement.classList += " " + mermaidContainer.classList;
|
||||
const hasStretchClass = mermaidContainer.classList.contains("r-stretch");
|
||||
if(hasStretchClass) {
|
||||
svgElement.style.width = "";
|
||||
svgElement.style.height = "";
|
||||
svgElement.style.maxWidth = "";
|
||||
svgElement.style.maxHeight = "";
|
||||
svgElement.style.minWidth = "";
|
||||
svgElement.style.minHeight = "";
|
||||
}
|
||||
for (const prop of mermaidContainer.style) {
|
||||
const value = mermaidContainer.style.getPropertyValue(prop);
|
||||
if (value && value.trim() !== '') {
|
||||
svgElement.style.setProperty(prop, value);
|
||||
}
|
||||
}
|
||||
mermaidContainer.remove();
|
||||
|
||||
// 4. Handle Overflow
|
||||
if(options.overflowVisible) {
|
||||
let selector = options.overflowVisible === '*' ? '*' : 'foreignObject';
|
||||
svgElement.querySelectorAll(selector).forEach((obj) => {
|
||||
obj.setAttribute('overflow', 'visible');
|
||||
});
|
||||
}
|
||||
|
||||
// 5. Apply Custom CSS Logic
|
||||
if (options.css.enabled && renderRules.length > 0) {
|
||||
const svgNS = "http://www.w3.org/2000/svg";
|
||||
|
||||
renderRules.forEach(({ selector, assignment, type }) => {
|
||||
try {
|
||||
selector = selector.trim();
|
||||
let indexPart = null;
|
||||
if(/.*\[\d+]$/.exec(selector)) {
|
||||
const digitPart = /\[\d+]$/.exec(selector)[0];
|
||||
indexPart = parseInt(digitPart.match(/\d+/)[0]);
|
||||
selector = selector.replace(/\[\d+]$/, '');
|
||||
}
|
||||
|
||||
const targets = svgElement.querySelectorAll(selector);
|
||||
targets.forEach((el, idx) => {
|
||||
if(indexPart !== null && idx + 1 !== indexPart)
|
||||
return;
|
||||
|
||||
if(options.css.debug) {
|
||||
console.log(selector, idx + 1, type, el)
|
||||
}
|
||||
|
||||
let modificationTarget = el;
|
||||
|
||||
// --- LOGIC FOR ARROW TYPES ---
|
||||
if (type === '^->') {
|
||||
// 1. OUTER WRAPPER
|
||||
// Wraps the element in a new <g> and modifies the <g>
|
||||
const wrapper = document.createElementNS(svgNS, 'g');
|
||||
el.parentNode.insertBefore(wrapper, el);
|
||||
wrapper.appendChild(el);
|
||||
modificationTarget = wrapper;
|
||||
|
||||
} else if (type === '_->') {
|
||||
// 2. INNER WRAPPER
|
||||
// Creates a new <g> inside the element, moves all children into it,
|
||||
// and modifies that new internal <g>
|
||||
const innerGroup = document.createElementNS(svgNS, 'g');
|
||||
|
||||
// Move all existing children of 'el' into 'innerGroup'
|
||||
while (el.firstChild) {
|
||||
innerGroup.appendChild(el.firstChild);
|
||||
}
|
||||
|
||||
el.appendChild(innerGroup);
|
||||
modificationTarget = innerGroup;
|
||||
}
|
||||
// Default '->' falls through here, keeping modificationTarget = el
|
||||
|
||||
const tokens = assignment.match(/(\[.+?\])|(\S+)/g) || [];
|
||||
|
||||
tokens.forEach(token => {
|
||||
if (token.startsWith('.')) {
|
||||
modificationTarget.classList.add(token.substring(1));
|
||||
} else if (token.startsWith('!.')) {
|
||||
modificationTarget.classList.remove(token.substring(2));
|
||||
} else if (token.startsWith('#')) {
|
||||
modificationTarget.id = token.substring(1);
|
||||
} else if (token.startsWith('[')) {
|
||||
const content = token.substring(1, token.length - 1);
|
||||
const eqIndex = content.indexOf('=');
|
||||
|
||||
if (eqIndex > -1) {
|
||||
const key = content.substring(0, eqIndex).trim();
|
||||
let val = content.substring(eqIndex + 1).trim();
|
||||
|
||||
if ((val.startsWith('"') && val.endsWith('"')) ||
|
||||
(val.startsWith("'") && val.endsWith("'"))) {
|
||||
val = val.substring(1, val.length - 1);
|
||||
}
|
||||
|
||||
modificationTarget.setAttribute(key, val);
|
||||
} else {
|
||||
modificationTarget.setAttribute(content, '');
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
} catch (e) {
|
||||
console.warn(`Mermaid Plugin: Failed to apply selector "${selector}".`, e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if(options.css.enabled && options.css.cssIndices) {
|
||||
const cssSelector = '[class*="' + options.css.indexClassPrefix + '"]';
|
||||
const fragmentsWithCssIndex = svgElement.querySelectorAll(cssSelector);
|
||||
|
||||
for (let fragment of fragmentsWithCssIndex) {
|
||||
let s = fragment.getAttribute('class');
|
||||
s = s.substring(
|
||||
s.indexOf(options.css.indexClassPrefix) + options.css.indexClassPrefix.length
|
||||
);
|
||||
s = s.substring(0, Math.max(s.indexOf(' '), s.length));
|
||||
fragment.classList.add('fragment');
|
||||
fragment.setAttribute('data-fragment-index', s);
|
||||
}
|
||||
}
|
||||
|
||||
if(hasStretchClass)
|
||||
svgElement.style.height = '100%';
|
||||
});
|
||||
|
||||
await Promise.all(renderPromises);
|
||||
reveal.layout();
|
||||
}
|
||||
};
|
||||
Binary file not shown.
@@ -0,0 +1,230 @@
|
||||
# Celery
|
||||
<!-- .slide: data-background-image="./assets/celery.jpg" -->
|
||||
|
||||
---
|
||||
|
||||
# Plan
|
||||
|
||||
- **Qu'est ce que Celery**
|
||||
- Tâche
|
||||
- Worker
|
||||
- Broker et Queue
|
||||
- Lancer une tâche
|
||||
- Débugger Celery
|
||||
- La parallélisation et les sous-tâches
|
||||
- Les barres de progression dans Celery
|
||||
|
||||
|
||||
|
||||
## Qu'est ce que Celery
|
||||
|
||||
<img src="./assets/celery_overview.svg">
|
||||
|
||||
|
||||
|
||||
### Tâche
|
||||
|
||||
```python
|
||||
from celery import shared_task
|
||||
|
||||
@shared_task(name="ptf.tasks.test_task")
|
||||
def test_task():
|
||||
print("toto")
|
||||
```
|
||||
Une tâche instanciée contient des métadonnées
|
||||
|
||||
> /!\ Faire la distinction entre tâche "instanciée" et tâche "déclarée"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
### Worker
|
||||
|
||||
```bash
|
||||
$ celery -A ptf_tools worker
|
||||
```
|
||||
|
||||
|
||||
|
||||
### Broker et Queue
|
||||
|
||||
```bash
|
||||
$ systemctl status rabbitmq-server.service
|
||||
● rabbitmq-server.service - RabbitMQ Messaging Server
|
||||
Loaded: loaded (/lib/systemd/system/rabbitmq-server.service; enabled; preset: enabled)
|
||||
Active: active (running) since Sun 2025-09-07 06:32:54 CEST; 6 months 21 days ago
|
||||
Main PID: 2939178 (beam.smp)
|
||||
Tasks: 36 (limit: 11885)
|
||||
Memory: 149.3M
|
||||
CPU: 1d 13h 28min 48.791s
|
||||
CGroup: /system.slice/rabbitmq-server.service
|
||||
├─2939178 /usr/lib/erlang/erts-13.1.5/bin/beam.smp -W w -MBas ageffcbf -MHas ageffcbf -MBlmbcs 512 -MHlmbcs 512 -MMmcs 30 -P 1048576 -t 5000000 -stbt db -zdbbl 128000 -sbwt none -sbwtdcpu none -sbwtdio none -- -root /usr/lib/erlang -bindir /usr/lib/erlang/erts-13.1.5/bin -progname erl -- -home /var/lib/>
|
||||
├─2939188 erl_child_setup 65536
|
||||
├─2939260 /usr/lib/erlang/erts-13.1.5/bin/inet_gethost 4
|
||||
├─2939261 /usr/lib/erlang/erts-13.1.5/bin/inet_gethost 4
|
||||
└─2939264 /bin/sh -s rabbit_disk_monitor
|
||||
|
||||
Sep 07 06:32:49 mathdoc-new-trammel.u-ga.fr systemd[1]: Starting rabbitmq-server.service - RabbitMQ Messaging Server...
|
||||
Sep 07 06:32:54 mathdoc-new-trammel.u-ga.fr systemd[1]: Started rabbitmq-server.service - RabbitMQ Messaging Server.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
# Plan
|
||||
|
||||
- Qu'est ce que Celery
|
||||
- **Lancer une tâche**
|
||||
- Exemple simple
|
||||
- Derrière les rideaux...
|
||||
- Débugger Celery
|
||||
- La parallélisation et les sous-tâches
|
||||
- Les barres de progression dans Celery
|
||||
|
||||
|
||||
|
||||
### Exemple simple
|
||||
Prérequis : un worker et un broker lancés
|
||||
```python
|
||||
from celery import shared_task
|
||||
|
||||
@shared_task(name="crawler.tasks.test_task")
|
||||
def test_task(self):
|
||||
print("TEST TASK")
|
||||
|
||||
# synchrone dans le thread actuel
|
||||
test_task()
|
||||
|
||||
# asynchrone dans un worker
|
||||
test_task.delay()
|
||||
```
|
||||
> /!\ Pour le worker et django, les fichiers de code sont les mêmes...
|
||||
|
||||
|
||||
|
||||
### Derrière les rideaux...
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
%%{init: {'theme': 'light', 'themeVariables': { 'darkMode': false }}}%%
|
||||
sequenceDiagram
|
||||
actor User
|
||||
participant Django
|
||||
participant Broker@{ "type" : "database" }
|
||||
participant Worker
|
||||
participant Postgres@{ "type" : "database" }
|
||||
User->>Django: Click button
|
||||
Django->>Broker: Send message : schedule task
|
||||
Broker->>Worker: Acquire task
|
||||
activate Worker
|
||||
Worker->>Postgres: Save results
|
||||
deactivate Worker
|
||||
User ->>Django: Query task result
|
||||
Django->>Postgres: Query task result
|
||||
Postgres-->>Django: task result
|
||||
Django-->>User: task result
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
%%{init: {'theme': 'light', 'themeVariables': { 'darkMode': false }}}%%
|
||||
sequenceDiagram
|
||||
actor User
|
||||
participant Django
|
||||
participant Worker
|
||||
participant Postgres@{ "type" : "database" }
|
||||
User->>Django: Click button
|
||||
Django->>()Worker: Schedule task
|
||||
activate Worker
|
||||
Worker->>Postgres: Save results
|
||||
deactivate Worker
|
||||
User ->>Django: Query task result
|
||||
Django->>Postgres: Query task result
|
||||
Postgres-->>Django: task result
|
||||
Django-->>User: task result
|
||||
</script>
|
||||
</div>
|
||||
---
|
||||
|
||||
# Plan
|
||||
|
||||
- Qu'est ce que Celery
|
||||
- Lancer une tâche
|
||||
- **Débugger Celery**
|
||||
- Lister les worker disponibles
|
||||
- Lister les tâches réalisables
|
||||
- Flower
|
||||
- Concepts avancés
|
||||
- La parallélisation et les sous-tâches
|
||||
- Les barres de progression dans Celery
|
||||
|
||||
|
||||
|
||||
### Lister les workers disponibles
|
||||
|
||||
`celery -A ptf_tools status`
|
||||
|
||||
```bash
|
||||
env PYTHONPATH=/var/www/ptf_tools/current/src/ /var/www/ptf_tools/shared/venv/bin/celery -A ptf_tools status
|
||||
```
|
||||
|
||||
```bash
|
||||
$ celery -A ptf_tools status
|
||||
-> executor@mathdoc-new-trammel.u-ga.fr: OK
|
||||
-> default@mathdoc-new-trammel.u-ga.fr: OK
|
||||
-> coordinator@mathdoc-new-trammel.u-ga.fr: OK
|
||||
3 nodes online.
|
||||
```
|
||||
|
||||
|
||||
|
||||
### Lister les tâches réalisables
|
||||
|
||||
`celery -A ptf_tools inspect registered`
|
||||
|
||||
```bash
|
||||
env PYTHONPATH=/var/www/ptf_tools/current/src/ /var/www/ptf_tools/shared/venv/bin/celery -A ptf_tools inspect registered
|
||||
```
|
||||
|
||||
```bash
|
||||
celery -A ptf_tools inspect registered
|
||||
-> executor@mathdoc-new-trammel.u-ga.fr: OK
|
||||
* ptf_tools.tasks.archiving_tasks.archive_numdam_collection
|
||||
* ptf_tools.tasks.archiving_tasks.archive_numdam_collections
|
||||
* ptf_tools.tasks.archiving_tasks.archive_numdam_resource
|
||||
* ptf_tools.tasks.archiving_tasks.archive_trammel_collection
|
||||
* ptf_tools.tasks.archiving_tasks.archive_trammel_collections
|
||||
* task.tasks.archiving_tasks.archive_collection
|
||||
* task.tasks.archiving_tasks.archive_collections
|
||||
* task.tasks.archiving_tasks.archive_resource
|
||||
* task.tasks.archiving_tasks.check_archive
|
||||
* task.tasks.increment_progress
|
||||
* task.tasks.tex_tasks.convert_article_tex
|
||||
* task.tasks.tex_tasks.remote_tex_to_xml
|
||||
* task.tasks.tex_tasks.update_article_body_from_xml
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Flower
|
||||
|
||||
```bash
|
||||
env FLOWER_UNAUTHENTICATED_API=true celery -A gdml flower --debug --broker_api="http://guest:guest@localhost:45253/api/ptf_host"
|
||||
```
|
||||
|
||||
http://localhost:5555/
|
||||
|
||||
<img src="./assets/flower.png">
|
||||
|
||||
|
||||
|
||||
## Concepts avancés
|
||||
- AsyncResult
|
||||
- Signature
|
||||
- Chain
|
||||
- Group
|
||||
- Chord
|
||||
|
||||
https://docs.celeryq.dev/en/main/reference/celery.html
|
||||
@@ -0,0 +1,412 @@
|
||||
# Plan
|
||||
|
||||
- Qu'est ce que Celery
|
||||
- Lancer une tâche
|
||||
- Débugger Celery
|
||||
- **La parallélisation et les sous-tâches**
|
||||
- Worker pool
|
||||
- Coordinator/Executor
|
||||
- Les barres de progression dans Celery
|
||||
|
||||
|
||||
|
||||
## Worker pool
|
||||
https://docs.celeryq.dev/en/main/userguide/workers.html#concurrency
|
||||
- parallélisation dans un seul worker
|
||||
- config: multiprocessing, threads ou coroutines
|
||||
|
||||
> Besoin de déclarer une limite max de sous-processus/threads
|
||||
|
||||
|
||||
|
||||
|
||||
### Cas séquentiel
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
block
|
||||
columns 1
|
||||
block:macro_task
|
||||
columns 7
|
||||
space:3 macro_task_label["macro_task"] space:3
|
||||
s(("start")) w1["work"] w2["work"] w3["work"] w4["work"] w5["work"] e(("end"))
|
||||
s-->w1
|
||||
w1-->w2
|
||||
w2-->w3
|
||||
w3-->w4
|
||||
w4-->w5
|
||||
w5-->e
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
class macro_task_label transparent
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
%%{init: { 'block': {'useMaxWidth':false, 'useWidth':1200}, 'theme':'neutral'} }%%
|
||||
block
|
||||
columns 1
|
||||
block:macro_task
|
||||
columns 1
|
||||
worker_label["worker"]
|
||||
block
|
||||
columns 1
|
||||
w1_label["thread1"]
|
||||
w1["macro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
w2_label["thread2"]
|
||||
w2["💤"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
w3_label["thread3"]
|
||||
w3["💤"]
|
||||
end
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef task fill:#ffffded9;
|
||||
class worker_label,w1_label,w2_label,w2,w3_label,w3 transparent
|
||||
class w1 task
|
||||
</script>
|
||||
</div>
|
||||
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
%%{init: { 'theme':'forest'} }%%
|
||||
block
|
||||
columns 1
|
||||
block:queue
|
||||
columns 1
|
||||
label["queue"]
|
||||
space:5
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef task fill:#ffffded9;
|
||||
class label,w1_label,w2_label,w2,w3_label,w3 transparent
|
||||
class w1 task
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
### Cas parallèle
|
||||
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
block
|
||||
columns 1
|
||||
block:macro_task
|
||||
columns 3
|
||||
macro_task_label["macro_task"]:3
|
||||
space
|
||||
block:w1
|
||||
columns 1
|
||||
w1_label["micro_task"]
|
||||
w1_["work"]
|
||||
end
|
||||
space
|
||||
s(("start"))
|
||||
block:w2
|
||||
columns 1
|
||||
w2_label["micro_task"]
|
||||
w2_["work"]
|
||||
end
|
||||
e(("end"))
|
||||
space
|
||||
block:w3
|
||||
columns 1
|
||||
w3_label["micro_task"]
|
||||
w3_["work"]
|
||||
end
|
||||
end
|
||||
s-->w1
|
||||
s-->w2
|
||||
s-->w3
|
||||
w1-->e
|
||||
w2-->e
|
||||
w3-->e
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef lg fill:lightgray,stroke:none;
|
||||
class macro_task_label,w1_label,w2_label,w3_label,w4_label,w5_label,s_ transparent
|
||||
class w1,w2,w3,w4,w5 lg
|
||||
</script>
|
||||
</div>
|
||||
|
||||
<div data-mermaid >
|
||||
<script type="text/mermaid">
|
||||
---
|
||||
config:
|
||||
theme: neutral
|
||||
---
|
||||
block
|
||||
columns 1
|
||||
block:macro_task
|
||||
columns 1
|
||||
worker_label["worker"]
|
||||
block
|
||||
columns 1
|
||||
w1_label["thread1"]
|
||||
w1["macro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
w2_label["thread2"]
|
||||
w2["micro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
w3_label["thread3"]
|
||||
w3["micro_task"]
|
||||
end
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef task fill:#ffffded9;
|
||||
class worker_label,w1_label,w2_label,w3_label transparent
|
||||
class w1 task
|
||||
</script>
|
||||
</div>
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
---
|
||||
config:
|
||||
theme: forest
|
||||
---
|
||||
block
|
||||
columns 1
|
||||
block:queue
|
||||
columns 1
|
||||
label["queue"]
|
||||
w1["micro_task"]
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef task fill:#eee,stroke:#999;
|
||||
class label transparent
|
||||
class w1 task
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
### Cas parallèle
|
||||
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
block
|
||||
columns 1
|
||||
block:macro_task
|
||||
columns 3
|
||||
macro_task_label["macro_task"]:3
|
||||
space
|
||||
block:w1
|
||||
columns 1
|
||||
w1_label["micro_task"]
|
||||
w1_["work"]
|
||||
end
|
||||
space
|
||||
s(("start"))
|
||||
block:w2
|
||||
columns 1
|
||||
w2_label["micro_task"]
|
||||
w2_["work"]
|
||||
end
|
||||
e(("end"))
|
||||
space
|
||||
block:w3
|
||||
columns 1
|
||||
w3_label["micro_task"]
|
||||
w3_["work"]
|
||||
end
|
||||
end
|
||||
s-->w1
|
||||
s-->w2
|
||||
s-->w3
|
||||
w1-->e
|
||||
w2-->e
|
||||
w3-->e
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef lg fill:lightgray,stroke:none;
|
||||
class macro_task_label,w1_label,w2_label,w3_label,w4_label,w5_label,s_ transparent
|
||||
class w1,w2,w3,w4,w5 lg
|
||||
</script>
|
||||
</div>
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
---
|
||||
config:
|
||||
theme: neutral
|
||||
---
|
||||
block
|
||||
columns 1
|
||||
block:macro_task
|
||||
columns 1
|
||||
worker_label["worker"]
|
||||
block
|
||||
columns 1
|
||||
w1_label["thread1"]
|
||||
w1["macro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
w2_label["thread2"]
|
||||
w2["macro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
w3_label["thread3"]
|
||||
w3["macro_task"]
|
||||
end
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef task fill:#ffffded9;
|
||||
class worker_label,w1_label,w2_label,w3_label transparent
|
||||
class w1,w2,w3 task
|
||||
</script>
|
||||
</div>
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
---
|
||||
config:
|
||||
theme: forest
|
||||
---
|
||||
block
|
||||
columns 1
|
||||
block:queue
|
||||
columns 1
|
||||
label["queue"]
|
||||
w1["micro_task"]
|
||||
w2["micro_task"]
|
||||
w3["micro_task"]
|
||||
w4["micro_task"]
|
||||
w5["micro_task"]
|
||||
w6["micro_task"]
|
||||
w7["micro_task"]
|
||||
w8["micro_task"]
|
||||
w9["micro_task"]
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef task fill:#eee,stroke:#999;
|
||||
class label transparent
|
||||
class w1,w2,w3,w4,w5,w6,w7,w8,w9 task
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
### Coordinator/Executor
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
---
|
||||
config:
|
||||
theme: neutral
|
||||
---
|
||||
block
|
||||
columns 1
|
||||
block:macro_task
|
||||
columns 1
|
||||
worker_label["coordinator worker"]
|
||||
block
|
||||
columns 1
|
||||
w1_label["thread1"]
|
||||
w1["macro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
w2_label["thread2"]
|
||||
w2["macro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
w3_label["thread3"]
|
||||
w3["macro_task"]
|
||||
end
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef task fill:#ffffded9;
|
||||
class worker_label,w1_label,w2_label,w3_label transparent
|
||||
class w1,w2,w3 task
|
||||
</script>
|
||||
</div>
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
---
|
||||
config:
|
||||
theme: forest
|
||||
---
|
||||
block
|
||||
columns 1
|
||||
block:queue
|
||||
columns 1
|
||||
label["coordinator queue"]
|
||||
w1["macro_task"]
|
||||
w2["macro_task"]
|
||||
w3["macro_task"]
|
||||
space:5
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef task fill:#ffffded9,stroke:#999;
|
||||
class label transparent
|
||||
class w1,w2,w3,w4,w5,w6,w7,w8,w9 task
|
||||
</script>
|
||||
</div>
|
||||
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
---
|
||||
config:
|
||||
theme: neutral
|
||||
---
|
||||
block
|
||||
columns 1
|
||||
block:macro_task
|
||||
columns 1
|
||||
worker_label["executor worker"]
|
||||
block
|
||||
columns 1
|
||||
w1_label["thread1"]
|
||||
w1["micro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
w2_label["thread2"]
|
||||
w2["micro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
w3_label["thread3"]
|
||||
w3["micro_task"]
|
||||
end
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef task fill:#eee,stroke:#999;
|
||||
class worker_label,w1_label,w2_label,w3_label transparent
|
||||
class w1,w2,w3 task
|
||||
</script>
|
||||
</div>
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
---
|
||||
config:
|
||||
theme: forest
|
||||
---
|
||||
block
|
||||
columns 1
|
||||
block:queue
|
||||
columns 1
|
||||
label["executor queue"]
|
||||
w1["micro_task"]
|
||||
w2["micro_task"]
|
||||
w3["micro_task"]
|
||||
w4["micro_task"]
|
||||
w5["micro_task"]
|
||||
w6["micro_task"]
|
||||
space:2
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef task fill:#eee,stroke:#999;
|
||||
class label transparent
|
||||
class w1,w2,w3,w4,w5,w6,w7,w8,w9 task
|
||||
</script>
|
||||
</div>
|
||||
@@ -0,0 +1,226 @@
|
||||
# Les barres de progression dans Celery
|
||||
<img src="./assets/progressbar.png">
|
||||
|
||||
- **Intro**
|
||||
- Communication Worker->Django
|
||||
- Communication Django->Client
|
||||
- Accès concurrent
|
||||
|
||||
|
||||
|
||||
## Intro
|
||||
|
||||
### Liste des courses
|
||||
- Progression en temps réel
|
||||
- Barre de progression propre à chaque tâche instanciée
|
||||
|
||||
### Contraintes
|
||||
- Tâches possiblement longues
|
||||
- Éviter le polling (fonctionner sur évènements)
|
||||
- Pas de résidus en cas de plantage
|
||||
|
||||
|
||||
|
||||
### Situation initiale
|
||||
|
||||
- Fonctionnalité non existante dans Celery
|
||||
- Solutions sur le net : progression par comptage de micro-tâches
|
||||
- Une bibliothèque existante sur le net : celery-progress
|
||||
|
||||
|
||||
|
||||
### Complexité supplémentaire
|
||||
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
%%{init: {'theme': 'light', 'themeVariables': { 'darkMode': false }}}%%
|
||||
sequenceDiagram
|
||||
actor User
|
||||
participant Django
|
||||
participant Broker@{ "type" : "database" }
|
||||
participant Worker
|
||||
participant Postgres@{ "type" : "database" }
|
||||
User->>Django: Click button
|
||||
Django->>Broker: Send message : schedule task
|
||||
Broker->>Worker: Acquire task
|
||||
activate Worker
|
||||
Worker->>Postgres: Save results
|
||||
deactivate Worker
|
||||
User ->>Django: Query task result
|
||||
Django->>Postgres: Query task result
|
||||
Postgres-->>Django: task result
|
||||
Django-->>User: task result
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div data-mermaid data-auto-animate>
|
||||
<script type="text/mermaid">
|
||||
sequenceDiagram
|
||||
actor User
|
||||
participant Django
|
||||
participant Broker@{ "type" : "database" }
|
||||
participant Worker
|
||||
participant Postgres@{ "type" : "database" }
|
||||
User->>Django: Click button
|
||||
Django->>Broker: Send message : schedule task
|
||||
Broker->>Worker: Acquire task
|
||||
activate Worker
|
||||
rect rgb(255, 224, 224)
|
||||
loop
|
||||
Worker->>Worker: Work
|
||||
Worker->>Broker: Progress event
|
||||
Broker->>Django: Progress event
|
||||
Django->>User: Progress event
|
||||
end
|
||||
end
|
||||
Worker->>Postgres: Save results
|
||||
deactivate Worker
|
||||
User ->>Django: Query task result
|
||||
Django->>Postgres: Query task result
|
||||
Postgres-->>Django: task result
|
||||
Django-->>User: task result
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<div data-mermaid >
|
||||
<script type="text/mermaid">
|
||||
sequenceDiagram
|
||||
actor User
|
||||
participant Django
|
||||
participant Worker
|
||||
participant Postgres@{ "type" : "database" }
|
||||
User->>Django: Click button
|
||||
Django->>()Worker: Send message : schedule task
|
||||
activate Worker
|
||||
rect rgb(255, 224, 224)
|
||||
loop
|
||||
Worker->>Worker: Work
|
||||
Worker->>()Django: Progress event
|
||||
Django->>User: Progress event
|
||||
end
|
||||
end
|
||||
Worker->>Postgres: Save results
|
||||
deactivate Worker
|
||||
User ->>Django: Query task result
|
||||
Django->>Postgres: Query task result
|
||||
Postgres-->>Django: task result
|
||||
Django-->>User: task result
|
||||
</script>
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## Communication worker -> server
|
||||
|
||||
- Coté Worker
|
||||
- Coté Django
|
||||
- Accès concurrent
|
||||
|
||||
---
|
||||
|
||||
### Coté Worker
|
||||
https://docs.celeryq.dev/en/main/reference/celery.events.html
|
||||
> The worker has the ability to send a message whenever some event happens. These events are then captured by tools like Flower, and celery events to monitor the cluster.
|
||||
|
||||
|
||||
|
||||
### Coté Worker
|
||||
<!-- .slide: data-auto-animate -->
|
||||
|
||||
```python[]
|
||||
class PtfTask(AbortableTask):
|
||||
_progress_dispatcher = None
|
||||
|
||||
@property
|
||||
def progress_dispatcher(self):
|
||||
# Singleton
|
||||
...
|
||||
|
||||
def update_state(self, task_id, state, meta, **kwargs):
|
||||
self.progress_dispatcher.send(
|
||||
"ptf-task-progress",
|
||||
data=meta,
|
||||
uuid=task_id or self.request.id,
|
||||
)
|
||||
return super().update_state(task_id, state, meta, **kwargs)
|
||||
|
||||
```
|
||||
<!-- .element.code: data-id="code" -->
|
||||
> On envoie un évènement `ptf-task-progress` quand on met à jour l'état de la tâche
|
||||
|
||||
|
||||
|
||||
### Coté Worker
|
||||
<!-- .slide: data-auto-animate -->
|
||||
```python[13]
|
||||
class PtfTask(AbortableTask):
|
||||
_progress_dispatcher = None
|
||||
|
||||
@property
|
||||
def progress_dispatcher(self):
|
||||
# Singleton
|
||||
...
|
||||
|
||||
def update_state(self, task_id, state, meta, **kwargs):
|
||||
self.progress_dispatcher.send(
|
||||
"ptf-task-progress",
|
||||
data=meta,
|
||||
uuid=task_id or self.request.id,
|
||||
)
|
||||
return super().update_state(task_id, state, meta, **kwargs)
|
||||
|
||||
```
|
||||
<!-- .element: data-id="code" -->
|
||||
> Point important: on peut faire avancer la progression de n'importe quelle tâche tant qu'on a son ID!
|
||||
|
||||
|
||||
|
||||
### Coté Worker
|
||||
|
||||
```python
|
||||
@shared_task(name="crawler.tasks.toto", base=PtfTask, bind=True)
|
||||
def print_toto(self, toto = "toto"):
|
||||
|
||||
# Initialize progress bar at 0
|
||||
self.update_state(
|
||||
meta={"current": index, "total": len(toto)},
|
||||
state=states.STARTED,
|
||||
)
|
||||
|
||||
for index, letter in enumerate(toto):
|
||||
# Do work
|
||||
print(letter)
|
||||
|
||||
# Update progress bar
|
||||
self.update_state(
|
||||
meta={"current": index, "total": len(toto)},
|
||||
state=states.STARTED,
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Coté Django
|
||||
|
||||
```python
|
||||
from celery import Celery, current_app
|
||||
|
||||
def send_sse_message(event):
|
||||
...
|
||||
|
||||
handlers = {
|
||||
"ptf-task-progress": send_sse_message,
|
||||
}
|
||||
|
||||
receiver = current_app.events.Receiver(
|
||||
channel=current_app.connection_for_read(), app=current_app, handlers=handlers
|
||||
)
|
||||
```
|
||||
> On envoie un message au client quand on reçoit un évènement `ptf-task-progress`
|
||||
@@ -0,0 +1,403 @@
|
||||
|
||||
## Accès concurrent
|
||||
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
block
|
||||
columns 1
|
||||
block:macro_task
|
||||
columns 6
|
||||
space:2 macro_task_label["macro_task"]:2 space:2
|
||||
s(("start")) w1["work"] u1["progress"] w2["work"] u2["progress"] e(("end"))
|
||||
s-->w1
|
||||
w1-->u1
|
||||
u1-->w2
|
||||
w2-->u2
|
||||
u2-->e
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
class macro_task_label transparent
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
---
|
||||
config:
|
||||
theme: neutral
|
||||
---
|
||||
block
|
||||
columns 1
|
||||
block:macro_task
|
||||
columns 1
|
||||
worker_label["worker"]
|
||||
block
|
||||
columns 1
|
||||
w1_label["thread1"]
|
||||
w1["macro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
w2_label["thread2"]
|
||||
w2["💤"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
w3_label["thread3"]
|
||||
w3["💤"]
|
||||
end
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef task fill:#ffffded9;
|
||||
class worker_label,w1_label,w2_label,w2,w3_label,w3 transparent
|
||||
class w1 task
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
## Accès concurrent
|
||||
Cas parallèle
|
||||
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
block
|
||||
columns 1
|
||||
block:macro_task
|
||||
columns 3
|
||||
macro_task_label["macro_task"]:3
|
||||
space
|
||||
block:w1
|
||||
columns 2
|
||||
w1_label["micro_task"]:2
|
||||
w1_["work"]
|
||||
u1["progress"]
|
||||
w1_-->u1
|
||||
end
|
||||
space
|
||||
s(("start"))
|
||||
block:w2
|
||||
columns 2
|
||||
w2_label["micro_task"]:2
|
||||
w2_["work"]
|
||||
u2["progress"]
|
||||
w2_-->u2
|
||||
end
|
||||
e(("end"))
|
||||
space
|
||||
block:w3
|
||||
columns 2
|
||||
w3_label["micro_task"]:2
|
||||
w3_["work"]
|
||||
u3["progress"]
|
||||
w3_-->u3
|
||||
end
|
||||
end
|
||||
s-->w1
|
||||
s-->w2
|
||||
s-->w3
|
||||
w1-->e
|
||||
w2-->e
|
||||
w3-->e
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef lg fill:lightgray,stroke:none;
|
||||
class macro_task_label,w1_label,w2_label,w3_label,w4_label,w5_label,s_ transparent
|
||||
class w1,w2,w3,w4,w5 lg
|
||||
</script>
|
||||
</div>
|
||||
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
---
|
||||
config:
|
||||
theme: neutral
|
||||
---
|
||||
block
|
||||
columns 2
|
||||
block:coordinator
|
||||
columns 1
|
||||
c_worker_label["coordinator"]
|
||||
block
|
||||
columns 1
|
||||
c_w1_label["thread1"]
|
||||
c_w1["macro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
c_w2_label["thread2"]
|
||||
c_w2["💤"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
c_w3_label["thread3"]
|
||||
c_w3["💤"]
|
||||
end
|
||||
end
|
||||
block:executor
|
||||
columns 1
|
||||
e_worker_label["executor"]
|
||||
block
|
||||
columns 1
|
||||
e_w1_label["thread1"]
|
||||
e_w1["micro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
e_w2_label["thread2"]
|
||||
e_w2["micro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
e_w3_label["thread3"]
|
||||
e_w3["micro_task"]
|
||||
end
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef task fill:#ffffded9;
|
||||
class c_worker_label,c_w1_label,c_w2_label,c_w3_label,c_w2,c_w3 transparent
|
||||
class e_worker_label,e_w1_label,e_w2_label,e_w3_label transparent
|
||||
class c_w1 task
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
## Accès concurrent
|
||||
Cas parallèle
|
||||
<!-- .slide: data-auto-animate -->
|
||||
<div data-mermaid >
|
||||
<script type="text/mermaid">
|
||||
flowchart LR
|
||||
micro_task_1[micro_task]
|
||||
micro_task_2[micro_task]
|
||||
macro_task
|
||||
micro_task_1-->|Get task progress|macro_task
|
||||
micro_task_2-->|Get task progress|macro_task
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
## Accès concurrent
|
||||
Cas parallèle
|
||||
<!-- .slide: data-auto-animate -->
|
||||
<div data-mermaid >
|
||||
<script type="text/mermaid">
|
||||
flowchart RL
|
||||
micro_task_1[micro_task]
|
||||
micro_task_2[micro_task]
|
||||
macro_task-->|2/10|micro_task_1
|
||||
macro_task-->|2/10|micro_task_2
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
## Accès concurrent
|
||||
Cas parallèle
|
||||
<!-- .slide: data-auto-animate -->
|
||||
<div data-mermaid >
|
||||
<script type="text/mermaid">
|
||||
flowchart LR
|
||||
micro_task_1[micro_task]
|
||||
micro_task_2[micro_task]
|
||||
micro_task_1-->|Set progress to 3/10|macro_task
|
||||
micro_task_2-->|Set progress to 3/10|macro_task
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
## Accès concurrent
|
||||
Retour au séquentiel
|
||||
<!-- .slide: data-auto-animate data-auto-animate-restart -->
|
||||
<div data-mermaid >
|
||||
<script type="text/mermaid">
|
||||
---
|
||||
config:
|
||||
theme: neutral
|
||||
---
|
||||
block
|
||||
columns 3
|
||||
block:coordinator
|
||||
columns 1
|
||||
c_worker_label["coordinator"]
|
||||
block
|
||||
columns 1
|
||||
c_w1_label["thread1"]
|
||||
c_w1["macro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
c_w2_label["thread2"]
|
||||
c_w2["💤"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
c_w3_label["thread3"]
|
||||
c_w3["💤"]
|
||||
end
|
||||
end
|
||||
block:executor
|
||||
columns 1
|
||||
e_worker_label["executor"]
|
||||
block
|
||||
columns 1
|
||||
e_w1_label["thread1"]
|
||||
e_w1["micro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
e_w2_label["thread2"]
|
||||
e_w2["micro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
e_w3_label["thread3"]
|
||||
e_w3["micro_task"]
|
||||
end
|
||||
end
|
||||
block:d
|
||||
columns 1
|
||||
d_worker_label["default"]
|
||||
block
|
||||
columns 1
|
||||
d_w1_label["thread1"]
|
||||
d_w1["💤"]
|
||||
end
|
||||
space:2
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef task fill:#ffffded9;
|
||||
class c_worker_label,c_w1_label,c_w2_label,c_w3_label,c_w2,c_w3 transparent
|
||||
class e_worker_label,e_w1_label,e_w2_label,e_w3_label transparent
|
||||
class d_worker_label,d_w1_label,d_w1 transparent
|
||||
class c_w1 task
|
||||
</script>
|
||||
</div>
|
||||
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
---
|
||||
config:
|
||||
theme: forest
|
||||
---
|
||||
block
|
||||
columns 1
|
||||
block:queue
|
||||
columns 1
|
||||
label["default queue"]
|
||||
space:6
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef task fill:#eee,stroke:#999;
|
||||
class label,w1_label,w2_label,w2,w3_label,w3 transparent
|
||||
class w1 task
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
## Accès concurrent
|
||||
Retour au séquentiel
|
||||
<!-- .slide: data-auto-animate -->
|
||||
<div data-mermaid >
|
||||
<script type="text/mermaid">
|
||||
---
|
||||
config:
|
||||
theme: neutral
|
||||
---
|
||||
block
|
||||
columns 3
|
||||
block:coordinator
|
||||
columns 1
|
||||
c_worker_label["coordinator"]
|
||||
block
|
||||
columns 1
|
||||
c_w1_label["thread1"]
|
||||
c_w1["macro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
c_w2_label["thread2"]
|
||||
c_w2["💤"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
c_w3_label["thread3"]
|
||||
c_w3["💤"]
|
||||
end
|
||||
end
|
||||
block:executor
|
||||
columns 1
|
||||
e_worker_label["executor"]
|
||||
block
|
||||
columns 1
|
||||
e_w1_label["thread1"]
|
||||
e_w1["micro_task"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
e_w2_label["thread2"]
|
||||
e_w2["💤"]
|
||||
end
|
||||
block
|
||||
columns 1
|
||||
e_w3_label["thread3"]
|
||||
e_w3["💤"]
|
||||
end
|
||||
end
|
||||
block:d
|
||||
columns 1
|
||||
d_worker_label["default"]
|
||||
block
|
||||
columns 1
|
||||
d_w1_label["thread1"]
|
||||
d_w1["progress"]
|
||||
end
|
||||
space:2
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef task fill:#ffffded9;
|
||||
class c_worker_label,c_w1_label,c_w2_label,c_w3_label,c_w2,c_w3 transparent
|
||||
class e_worker_label,e_w1_label,e_w2_label,e_w3_label,e_w2,e_w3 transparent
|
||||
class d_worker_label,d_w1_label transparent
|
||||
class c_w1 task
|
||||
</script>
|
||||
</div>
|
||||
|
||||
<div data-mermaid>
|
||||
<script type="text/mermaid">
|
||||
---
|
||||
config:
|
||||
theme: forest
|
||||
---
|
||||
block
|
||||
columns 1
|
||||
block:queue
|
||||
columns 1
|
||||
label["default queue"]
|
||||
w1["progress"]
|
||||
space:5
|
||||
end
|
||||
classDef transparent fill:none,stroke:none;
|
||||
classDef task fill:#eee,stroke:#999;
|
||||
class label,w1_label,w2_label,w2,w3_label,w3 transparent
|
||||
class w1 task
|
||||
</script>
|
||||
</div>
|
||||
|
||||
|
||||
---
|
||||
|
||||
|
||||
## Différences avec celery-progress
|
||||
|
||||
| | ptf-task | celery-progress |
|
||||
|----------------|------------------------------|---------------------------------|
|
||||
| Broker | RabbitMQ ou Redis | RabbitMQ ou Redis |
|
||||
| Backend | Celery events | Channel Layers |
|
||||
| | | |
|
||||
| Streaming | Django StreamingHttpResponse | Channel AsyncWebsocketConsumer |
|
||||
| Protocol | HTTP ou SSE | HTTP ou WebSocket |
|
||||
Reference in New Issue
Block a user