Fix parallel code gen in RT backend

• Properly use Arc<Mutex<_Option<JoinHandle<_, _>>_>> for task parallel calls.
  • Include Arc clones right before opening async closure (for async calls and for fork-joins).
• Make data parallel fork-joins not task parallel by default (can still be outlined for task parallelism).
• Fix edge case in function inlining.

hercules_cg/src/rt.rs

@@ -138,7 +138,6 @@ struct RTContext<'a> {
 #[derive(Debug, Clone, Default)]
 struct RustBlock {
     prologue: String,
-    arc_clones: String,
     data: String,
     phi_tmp_assignments: String,
     phi_assignments: String,
@@ -269,9 +268,8 @@ impl<'a> RTContext<'a> {
             } else {
                 write!(
                     w,
-                    "{}{}{}{}{}{}",
+                    "{}{}{}{}{}",
                     block.prologue,
-                    block.arc_clones,
                     block.data,
                     block.phi_tmp_assignments,
                     block.phi_assignments,
@@ -361,6 +359,16 @@ impl<'a> RTContext<'a> {
                     write!(prologue, " {{")?;
                 }
 
+                // Emit clones of arcs used inside the fork-join.
+                for other_id in (0..func.nodes.len()).map(NodeID::new) {
+                    if self.def_use.get_users(other_id).into_iter().any(|user_id| {
+                        self.nodes_in_fork_joins[&id].contains(&self.bbs.0[user_id.idx()])
+                    }) && let Some(arc) = self.clone_arc(other_id)
+                    {
+                        write!(prologue, "{}", arc)?;
+                    }
+                }
+
                 // Spawn an async closure and push its future to a Vec.
                 write!(
                     prologue,
@@ -569,13 +577,12 @@ impl<'a> RTContext<'a> {
                 // The device backends ensure that device functions have the
                 // same interface as AsyncRust functions.
                 let block = &mut blocks.get_mut(&bb).unwrap();
-                let arc_clones = &mut block.arc_clones;
                 let block = &mut block.data;
                 let is_async = func.schedules[id.idx()].contains(&Schedule::AsyncCall);
                 if is_async {
                     for arg in args {
                         if let Some(arc) = self.clone_arc(*arg) {
-                            write!(arc_clones, "{}", arc)?;
+                            write!(block, "{}", arc)?;
                         }
                     }
                 }
@@ -585,9 +592,9 @@ impl<'a> RTContext<'a> {
                         format!("{} = ", self.get_value(id, bb, true))
                     }
                     (_, true) => {
-                        write!(arc_clones, "{}", self.clone_arc(id).unwrap())?;
+                        write!(block, "{}", self.clone_arc(id).unwrap())?;
                         format!(
-                            "*async_call_{}.lock().await = Some(::async_std::task::spawn(async move {{ ",
+                            "*async_call_{}.lock().await = ::hercules_rt::__FutureSlotWrapper::new(::async_std::task::spawn(async move {{ ",
                             id.idx(),
                         )
                     }
@@ -1099,7 +1106,7 @@ impl<'a> RTContext<'a> {
             if is_async_call {
                 write!(
                     w,
-                    "let mut async_call_{} = ::std::sync::Arc::new(::async_std::sync::Mutex::new(None));",
+                    "let mut async_call_{} = ::std::sync::Arc::new(::async_std::sync::Mutex::new(::hercules_rt::__FutureSlotWrapper::empty()));",
                     idx,
                 )?;
             } else {
@@ -1375,10 +1382,7 @@ impl<'a> RTContext<'a> {
             && func.schedules[id.idx()].contains(&Schedule::AsyncCall)
         {
             assert!(!lhs);
-            format!(
-                "async_call_{}.lock().await.as_mut().unwrap().await",
-                id.idx(),
-            )
+            format!("async_call_{}.lock().await.inspect().await", id.idx(),)
         } else {
             format!("node_{}", id.idx())
         }