From 2f7c4db6e4970627c1b20d0a4bccc4bdfe8b12f1 Mon Sep 17 00:00:00 2001
From: Eduard Valeyev <eduard@valeyev.net>
Date: Mon, 10 Jun 2024 07:50:07 -0400
Subject: [PATCH] amend Fib<N examples: ttg::execute before sending data

---
 README.md                                     | 23 +++++++++++++++----
 doc/dox/dev/devsamp/fibonacci/fibonacci.cc    |  8 +++++--
 .../dev/devsamp/fibonacci/fibonacci_device.cc | 10 +++++---
 3 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 8eba0bd0b..bfeec13a0 100644
--- a/README.md
+++ b/README.md
@@ -145,7 +145,7 @@ The "Hello, World!" example contains a single TT that executes a single task (he
 
 ## Execute TTG
 
-To execute a TTG we must make it executable (this will declare the TTG complete). To execute the TTG its root TT must receive at least one message; since in this case the task does not receive either task ID or data the message is empty (i.e., void):
+To execute a TTG we must make it executable (this will declare the TTG  program complete so no additional changes to the flowgraph are possible). To execute the TTG its root TT must receive at least one message; since in this case the task does not receive either task ID or data the message is empty (i.e., void):
 
 ```cpp
   ttg::make_graph_executable(tt);
@@ -154,7 +154,7 @@ To execute a TTG we must make it executable (this will declare the TTG complete)
       tt->invoke();
 ```
 
-Note that we must ensure that only one such message must be generated. Since TTG execution uses the Single Program Multiple Data (SPMD) model,
+`ttg::execute()` must occur before, not after, sending any messages. Note also that we must ensure that only one such message must be generated. Since TTG execution uses the Single Program Multiple Data (SPMD) model,
 when launching the TTG program as multiple processes only the first process (rank) gets to send the message.
 
 ## Finalize TTG
@@ -303,12 +303,11 @@ int main(int argc, char* argv[]) {
 
   auto fib = make_ttg_fib_lt(N);
   ttg::make_graph_executable(fib.get());
+  ttg::execute();
   if (ttg::default_execution_context().rank() == 0)
     fib->template in<0>()->send(1, Fn{});;
 
-  ttg::execute();
   ttg::fence();
-
   ttg::finalize();
   return 0;
 }
@@ -397,6 +396,22 @@ auto make_ttg_fib_lt(const int64_t F_n_max = 1000) {
   ops.emplace_back(std::move(print));
   return make_ttg(std::move(ops), ins, std::make_tuple(), "Fib_n < N");
 }
+
+int main(int argc, char* argv[]) {
+  ttg::initialize(argc, argv, -1);
+  int64_t N = 1000;
+  if (argc > 1) N = std::atol(argv[1]);
+
+  auto fib = make_ttg_fib_lt(N);
+  ttg::make_graph_executable(fib.get());
+  ttg::execute();
+  if (ttg::default_execution_context().rank() == 0)
+    fib->template in<0>()->send(1, Fn{});;
+
+  ttg::fence();
+  ttg::finalize();
+  return 0;
+}
 ```
 
 Although the structure of the device-capable program is nearly identical to the CPU version, there are important differences:
diff --git a/doc/dox/dev/devsamp/fibonacci/fibonacci.cc b/doc/dox/dev/devsamp/fibonacci/fibonacci.cc
index d2d829c45..3de431979 100644
--- a/doc/dox/dev/devsamp/fibonacci/fibonacci.cc
+++ b/doc/dox/dev/devsamp/fibonacci/fibonacci.cc
@@ -47,12 +47,16 @@ int main(int argc, char* argv[]) {
   ttg::initialize(argc, argv, -1);
   int64_t N = (argc > 1) ? std::atol(argv[1]) : 1000;
 
+  // make TTG
   auto fib = make_ttg_fib_lt(N);
+  // program complete, declare it executable
   ttg::make_graph_executable(fib.get());
+  // start execution
+  ttg::execute();
+  // start the computation by sending the first message
   if (ttg::default_execution_context().rank() == 0)
     fib->template in<0>()->send(1, Fn{});;
-
-  ttg::execute();
+  // wait for the computation to finish
   ttg::fence();
 
   ttg::finalize();
diff --git a/doc/dox/dev/devsamp/fibonacci/fibonacci_device.cc b/doc/dox/dev/devsamp/fibonacci/fibonacci_device.cc
index a1603cb58..99dbc37ca 100644
--- a/doc/dox/dev/devsamp/fibonacci/fibonacci_device.cc
+++ b/doc/dox/dev/devsamp/fibonacci/fibonacci_device.cc
@@ -74,13 +74,17 @@ int main(int argc, char* argv[]) {
   ttg::trace_on();
   int64_t N = 1000;
   if (argc > 1) N = std::atol(argv[1]);
-  auto fib = make_ttg_fib_lt(N);  // computes largest F_n < N
 
+  // make TTG
+  auto fib = make_ttg_fib_lt(N);  // computes largest F_n < N
+  // program complete, declare it executable
   ttg::make_graph_executable(fib.get());
+  // start execution
+  ttg::execute(ttg::ttg_default_execution_context());
+  // start the computation by sending the first message
   if (ttg::default_execution_context().rank() == 0)
     fib->template in<0>()->send(1, Fn{});;
-
-  ttg::execute(ttg::ttg_default_execution_context());
+  // wait for the computation to finish
   ttg::fence(ttg::ttg_default_execution_context());
 
   ttg::finalize();