Skip to content

Commit 3b5535c

Browse files
jan-goraljan-goral
authored
feat: Implement tool for parallel execution (#2009)
Fixes #2001 ### Parallel execution algorithm * [README.md](https://github.com/Flank/flank/blob/2001_Implement_tool_for_parallel_execution/tool/execution/parallel/README.md) #### Example graph execution ![example-graph](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-example-graph.puml) #### Algorithm ![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-execution.puml) #### Functions ![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-execution-api-functions.puml) #### Types ![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-execution-api-structures.puml) ### Different graph states ![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/task-graph.puml) ## Test Plan > How do we know the code works? Unit tests are passing. ## Checklist - [x] Documented - [x] Unit tested
1 parent 33e946f commit 3b5535c

32 files changed

Lines changed: 2068 additions & 0 deletions

‎docs/hld/parallel-example-graph.puml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
@startuml
2+
'https://plantuml.com/component-diagram
3+
skinparam componentStyle rectangle
4+
5+
component Finalize
6+
7+
component NotifyRemoteServices
8+
component GenerateReport
9+
10+
component CalculateResults
11+
12+
component PerformExecution
13+
14+
component CollectRemoteData1
15+
component FetchPreviousResults
16+
17+
component InitialCalculations
18+
component RunRemoteProcess1
19+
component RunRemoteProcess2
20+
21+
component CollectLocalData2
22+
component CollectLocalData1
23+
component Authorize
24+
25+
Finalize *--> GenerateReport
26+
Finalize *--> NotifyRemoteServices
27+
28+
GenerateReport *--> CalculateResults
29+
NotifyRemoteServices *--> CalculateResults
30+
31+
CalculateResults *--> PerformExecution
32+
CalculateResults *--> FetchPreviousResults
33+
34+
InitialCalculations *--> CollectLocalData1
35+
InitialCalculations *--> CollectLocalData2
36+
37+
PerformExecution *--> CollectRemoteData1
38+
PerformExecution *---> InitialCalculations
39+
40+
FetchPreviousResults *--> Authorize
41+
42+
InitialCalculations -[hidden]r- RunRemoteProcess1
43+
44+
CollectRemoteData1 *--> RunRemoteProcess1
45+
CollectRemoteData1 *--> RunRemoteProcess2
46+
47+
RunRemoteProcess1 *--> Authorize
48+
RunRemoteProcess2 *--> Authorize
49+
RunRemoteProcess2 -[hidden]r- FetchPreviousResults
50+
51+
52+
@enduml

‎docs/hld/parallel-execution-api-functions.puml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
@startuml
2+
'left to right direction
3+
package "parallel" {
4+
5+
object "Parallel.Function.invoke()" as Function {
6+
creates
7+
}
8+
9+
class "Parallel.Task" as Task<R>
10+
class "Parallel.Task.Signature" as Task_Signature<R>
11+
interface "Parallel.Type<R>" as Type
12+
interface "Parallel.Type<R>" as Type2
13+
interface ExecuteTask<R> << (T, orchid) >>
14+
interface ParallelState << (T, orchid) >>
15+
16+
object "using" as usingType {
17+
creates
18+
}
19+
object "using" as usingSignature {
20+
creates
21+
}
22+
object from {
23+
creates
24+
}
25+
26+
object "invoke()" as invokeExecution {
27+
executes
28+
}
29+
object "invoke()" as reduce {
30+
reduces
31+
}
32+
33+
ExecuteTask <. "1" Function
34+
35+
ExecuteTask <--o "1" usingSignature
36+
usingSignature "1" #--> Task_Signature
37+
usingSignature "1" ..> Task
38+
usingSignature -right-* usingType
39+
40+
Type <--# "1" usingType
41+
ExecuteTask <--o "1" usingType
42+
usingType "1" ..> Task
43+
usingType *-left- from
44+
45+
Type <--# "1" from
46+
Type <--o "*" from
47+
from "1" ..> Task_Signature
48+
49+
Type2 <--o "*" reduce
50+
Task <. "*" reduce
51+
Task <-# "*" reduce
52+
53+
Task <--# "*" invokeExecution
54+
55+
invokeExecution "*" .left.> ParallelState
56+
invokeExecution "1" o-left-> ParallelState
57+
58+
}
59+
@enduml

‎docs/hld/parallel-execution-api-structures.puml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
@startuml
2+
package "parallel" {
3+
class "Parallel.Context" as Context {
4+
out: Output?
5+
}
6+
class "Parallel.Task" as Task<R> {
7+
signature: Signature<R>
8+
execute: ExecuteTask<R>
9+
}
10+
class "Parallel.Task.Signature" as Signature<R>{
11+
returns: Type<R>
12+
args: Set<Type<*>>
13+
}
14+
class "Parallel.Function" as Function<X: ParallelContext> {
15+
context: () -> X
16+
= suspend <R> (ExecuteTaskInContext) -> ExecuteTask
17+
}
18+
interface "Any data" as Data << (R, yellow) >>
19+
interface ExecuteTask<R> << (T, orchid) >> {
20+
= suspend ParallelState.() -> R
21+
}
22+
interface ExecuteTaskInContext<X: Parallel.Context> << (T, orchid) >> {
23+
= suspend X.() -> R
24+
}
25+
interface "Parallel.Type" as Type <T>
26+
interface ParallelState << (T, orchid) >> {
27+
= Map<Parallel.Type<*>, Any>
28+
}
29+
interface Output << (T, orchid) >> {
30+
= Any.() -> Unit
31+
}
32+
33+
Function ..> ExecuteTask
34+
Function #.. ExecuteTaskInContext
35+
36+
ExecuteTaskInContext #.. Context
37+
ExecuteTaskInContext .> Data
38+
39+
Data <. ExecuteTask
40+
Data "*" --o "1" ParallelState
41+
42+
ExecuteTask "1" -* "1" Task
43+
ExecuteTask #.. ParallelState
44+
45+
Context --|> Type
46+
Context "0..1" o..> "1" Output
47+
48+
ParallelState "1" -o "1" Context
49+
ParallelState "1" o- "*" Type
50+
51+
Task "1" *-- "1" Signature
52+
53+
Type "1" -* "1" Signature
54+
Type "*" -o "1" Signature
55+
56+
@enduml

‎docs/hld/parallel-execution.puml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
@startuml
2+
skinparam ConditionEndStyle hline
3+
4+
|Invoke execution|
5+
split
6+
-[hidden]->
7+
start
8+
:arguments;
9+
:initial state;
10+
repeat :Handle state}
11+
-[dotted]->
12+
if (is channel closed for receive?) then
13+
stop
14+
else (no)
15+
endif
16+
:next result<
17+
if (on first error) then (true)
18+
:Abort execution}
19+
else (false)
20+
endif
21+
repeat while (accumulate result)
22+
detach
23+
24+
|Handle state|
25+
split again
26+
-[hidden]->
27+
if (is complete) then (yes)
28+
:close channel/
29+
elseif (is finished) then (yes)
30+
:nothing to do;
31+
detach
32+
else
33+
:filter tasks for current state;
34+
:group them by arguments;
35+
repeat :prepare properties|
36+
repeat :Execute task}
37+
repeat while (for each task)
38+
repeat while (for each group)
39+
endif
40+
detach
41+
42+
|Execute task|
43+
split again
44+
-[hidden]->
45+
partition async {
46+
:log start;
47+
:prepare context;
48+
if (execute task in context) then (try)
49+
:log success;
50+
else (catch)
51+
:log exception;
52+
endif
53+
:send result]
54+
detach
55+
}
56+
57+
|Abort execution|
58+
split again
59+
-[hidden]->
60+
:cancel scope;
61+
repeat :send "not started"]
62+
repeat while (for each remaining task)
63+
:close channel/
64+
detach
65+
@enduml

‎docs/hld/task-graph.puml‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
@startuml
2+
package "Valid graph" {
3+
map H1 #lightgreen {
4+
}
5+
map I1 #lightgreen {
6+
}
7+
map F1 #lightgreen {
8+
I1 *--> I1
9+
}
10+
map E1 #lightgreen {
11+
I1 *--> I1
12+
}
13+
map D1 #lightgreen {
14+
I1 *--> I1
15+
}
16+
map C1 #lightgreen {
17+
D1 *--> D1
18+
}
19+
map B1 #lightgreen {
20+
C1 *-> C1
21+
E1 *--> E1
22+
H1*---> H1
23+
}
24+
map A1 #lightgreen {
25+
B1 *--> B1
26+
F1 *--> F1
27+
C1 *--> C1
28+
I1 *--> I1
29+
}
30+
}
31+
package "Selected graph" {
32+
map H2 #lightgreen {
33+
}
34+
map I2 #lightgreen {
35+
}
36+
map F2 #lightgrey {
37+
I2 *--> I2
38+
}
39+
map E2 #lightgreen {
40+
I2 *--> I2
41+
}
42+
map D2 #lightgreen {
43+
I2 *--> I2
44+
}
45+
map C2 #lightgreen {
46+
D2 *--> D2
47+
}
48+
map B2 #lightblue {
49+
C2 *-> C2
50+
E2 *--> E2
51+
H2*---> H2
52+
}
53+
map A2 #lightgrey {
54+
B2 *--> B2
55+
F2 *--> F2
56+
C2 *--> C2
57+
I2 *--> I2
58+
}
59+
}
60+
package "Broken graph" {
61+
map H3 #lightgreen {
62+
}
63+
map I3 #black {
64+
}
65+
map F3 #red {
66+
I3 *--> I3
67+
}
68+
map E3 #red {
69+
I3 *--> I3
70+
}
71+
map D3 #red {
72+
I3 *--> I3
73+
}
74+
map C3 #yellow {
75+
D3 *--> D3
76+
}
77+
map B3 #yellow {
78+
C3 *-> C3
79+
E3 *--> E3
80+
H3*---> H3
81+
}
82+
map A3 #yellow {
83+
B3 *--> B3
84+
F3 *--> F3
85+
C3 *--> C3
86+
I3 *--> I3
87+
}
88+
}
89+
90+
package "Failed graph" {
91+
map H4 #lightgreen {
92+
}
93+
map I4 #lightgreen {
94+
}
95+
map F4 #red {
96+
I4 *--> I4
97+
}
98+
map E4 #yellow {
99+
I4 *--> I4
100+
}
101+
map D4 #yellow {
102+
I4 *--> I4
103+
}
104+
map C4 #lightgrey {
105+
D4 *--> D4
106+
}
107+
map B4 #lightgrey {
108+
C4 *-> C4
109+
E4 *--> E4
110+
H4*---> H4
111+
}
112+
map A4 #lightgrey {
113+
B4 *--> B4
114+
F4 *--> F4
115+
C4 *--> C4
116+
I4 *--> I4
117+
}
118+
}
119+
@enduml

‎settings.gradle.kts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ include(
3131
":tool:junit",
3232
":tool:log",
3333
":tool:log:format",
34+
":tool:execution:parallel",
35+
":tool:execution:synchronized",
3436
)
3537

3638
plugins {

0 commit comments

Comments
 (0)