diff --git a/hercules_opt/src/ccp.rs b/hercules_opt/src/ccp.rs
index b626148c936e384b6bc0a9aaf951c35a9c4b4736..87d23c11aeb4759a8b1c288fb27ba33845f1dc18 100644
--- a/hercules_opt/src/ccp.rs
+++ b/hercules_opt/src/ccp.rs
@@ -697,6 +697,11 @@ fn ccp_flow_function(
}),
constant: ConstantLattice::bottom(),
},
+ // Data projections are uninterpretable.
+ Node::DataProjection { data, selection: _ } => CCPLattice {
+ reachability: inputs[data.idx()].reachability.clone(),
+ constant: ConstantLattice::bottom(),
+ },
Node::IntrinsicCall { intrinsic, args } => {
let mut new_reachability = ReachabilityLattice::bottom();
let mut new_constant = ConstantLattice::top();
@@ -961,8 +966,9 @@ fn ccp_flow_function(
constant: ConstantLattice::bottom(),
}
}
- // Projection handles reachability when following an if or match.
- Node::Projection { control, selection } => match &editor.func().nodes[control.idx()] {
+ // Control projection handles reachability when following an if or match.
+ Node::ControlProjection { control, selection } => match &editor.func().nodes[control.idx()]
+ {
Node::If { control: _, cond } => {
let cond_constant = &inputs[cond.idx()].constant;
let if_reachability = &inputs[control.idx()].reachability;
diff --git a/hercules_opt/src/editor.rs b/hercules_opt/src/editor.rs
index 16e5c3264d33a7c9bef85fc0fa3cec02963dbf48..51c2727556c855b2bda40c4ed7088b5a729d84bd 100644
--- a/hercules_opt/src/editor.rs
+++ b/hercules_opt/src/editor.rs
@@ -69,7 +69,7 @@ pub struct FunctionEdit<'a: 'b, 'b> {
// Compute a def-use map entries iteratively.
updated_def_use: BTreeMap<NodeID, HashSet<NodeID>>,
updated_param_types: Option<Vec<TypeID>>,
- updated_return_type: Option<TypeID>,
+ updated_return_types: Option<Vec<TypeID>>,
// Keep track of which deleted and added node IDs directly correspond.
sub_edits: Vec<(NodeID, NodeID)>,
}
@@ -208,7 +208,7 @@ impl<'a: 'b, 'b> FunctionEditor<'a> {
added_labels: Vec::new().into(),
updated_def_use: BTreeMap::new(),
updated_param_types: None,
- updated_return_type: None,
+ updated_return_types: None,
sub_edits: vec![],
};
@@ -228,7 +228,7 @@ impl<'a: 'b, 'b> FunctionEditor<'a> {
added_labels,
updated_def_use,
updated_param_types,
- updated_return_type,
+ updated_return_types,
sub_edits,
} = populated_edit;
@@ -358,8 +358,8 @@ impl<'a: 'b, 'b> FunctionEditor<'a> {
}
// Step 9: update return type if necessary.
- if let Some(return_type) = updated_return_type {
- editor.function.return_type = return_type;
+ if let Some(return_types) = updated_return_types {
+ editor.function.return_types = return_types;
}
true
@@ -768,6 +768,9 @@ impl<'a, 'b> FunctionEdit<'a, 'b> {
}
Type::Summation(tys) => Constant::Summation(id, 0, self.add_zero_constant(tys[0])),
Type::Array(_, _) => Constant::Array(id),
+ Type::MultiReturn(_) => {
+ panic!("PANIC: Can't create zero constant for multi-return types.")
+ }
};
self.add_constant(constant_to_construct)
}
@@ -791,6 +794,9 @@ impl<'a, 'b> FunctionEdit<'a, 'b> {
Type::Product(_) | Type::Summation(_) | Type::Array(_, _) => {
panic!("PANIC: Can't create one constant of a collection type.")
}
+ Type::MultiReturn(_) => {
+ panic!("PANIC: Can't create one constant for multi-return types.")
+ }
};
self.add_constant(constant_to_construct)
}
@@ -835,8 +841,8 @@ impl<'a, 'b> FunctionEdit<'a, 'b> {
self.updated_param_types = Some(tys);
}
- pub fn set_return_type(&mut self, ty: TypeID) {
- self.updated_return_type = Some(ty);
+ pub fn set_return_types(&mut self, tys: Vec<TypeID>) {
+ self.updated_return_types = Some(tys);
}
}
diff --git a/hercules_opt/src/fork_guard_elim.rs b/hercules_opt/src/fork_guard_elim.rs
index df40e60f89f0490cacb35d6e9754f3b134ed1483..c480f266f683cec84f0517db9842dd98566f22b9 100644
--- a/hercules_opt/src/fork_guard_elim.rs
+++ b/hercules_opt/src/fork_guard_elim.rs
@@ -95,7 +95,7 @@ fn guarded_fork(
});
// Whose predecessor is a read from an if
- let Node::Projection {
+ let Node::ControlProjection {
control: if_node,
ref selection,
} = function.nodes[control.idx()]
@@ -226,7 +226,7 @@ fn guarded_fork(
return None;
};
// Other predecessor needs to be the other projection from the guard's if
- let Node::Projection {
+ let Node::ControlProjection {
control: if_node2,
ref selection,
} = function.nodes[other_pred.idx()]
diff --git a/hercules_opt/src/inline.rs b/hercules_opt/src/inline.rs
index f01b2366e8da12be265b65ce29f6cb977ea7c618..2a5ad9afdf06f2591ea25c5b97e5bef0ceff2282 100644
--- a/hercules_opt/src/inline.rs
+++ b/hercules_opt/src/inline.rs
@@ -1,5 +1,4 @@
use std::collections::HashMap;
-use std::iter::zip;
use hercules_ir::callgraph::*;
use hercules_ir::def_use::*;
@@ -125,13 +124,25 @@ fn inline_func(
assert_eq!(call_pred.as_ref().len(), 1);
let call_pred = call_pred.as_ref()[0];
let called_func = called[&function].func();
+ let call_users = editor.get_users(id);
+ let call_projs = call_users
+ .map(|node_id| {
+ (
+ node_id,
+ editor.func().nodes[node_id.idx()]
+ .try_data_proj()
+ .expect("PANIC: Call user is not a data projection")
+ .1,
+ )
+ })
+ .collect::<Vec<_>>();
// We can't inline calls to functions with multiple returns.
let Some(called_return) = single_return_nodes[function.idx()] else {
continue;
};
let called_return_uses = get_uses(&called_func.nodes[called_return.idx()]);
let called_return_pred = called_return_uses.as_ref()[0];
- let called_return_data = called_return_uses.as_ref()[1];
+ let called_return_data = &called_return_uses.as_ref()[1..];
// Perform the actual edit.
editor.edit(|mut edit| {
@@ -209,8 +220,12 @@ fn inline_func(
}
}
- // Finally, delete the call node.
- edit = edit.replace_all_uses(id, old_id_to_new_id(called_return_data))?;
+ // Replace and delete the call's (data projection) users and the call node
+ for (proj_id, proj_idx) in call_proj {
+ edit =
+ edit.replace_all_uses(proj_id, old_id_to_new_id(called_return_data[proj_idx]))?;
+ edit = edit.delete_node(proj_id)?;
+ }
edit = edit.delete_node(control)?;
edit = edit.delete_node(id)?;
diff --git a/hercules_opt/src/loop_bound_canon.rs b/hercules_opt/src/loop_bound_canon.rs
index 680236f168c04fb26f8f8befd9ea865835235fca..a1ad625785aae6e0628cda3816a70f32aca43ef1 100644
--- a/hercules_opt/src/loop_bound_canon.rs
+++ b/hercules_opt/src/loop_bound_canon.rs
@@ -113,7 +113,7 @@ pub fn canonicalize_single_loop_bounds(
// FIXME: This is quite fragile.
let guard_info: Option<(NodeID, NodeID, NodeID, NodeID)> = (|| {
- let Node::Projection {
+ let Node::ControlProjection {
control,
selection: _,
} = editor.node(loop_pred)
diff --git a/hercules_opt/src/outline.rs b/hercules_opt/src/outline.rs
index 874e75e739b0f05f72f0f8cfa4c6ae6540ed9c6f..6a9b6084b7442299f8fe3c836bca25626eee7db6 100644
--- a/hercules_opt/src/outline.rs
+++ b/hercules_opt/src/outline.rs
@@ -180,12 +180,11 @@ pub fn outline(
editor.edit(|mut edit| {
// Step 2: assemble the outlined function.
let u32_ty = edit.add_type(Type::UnsignedInteger32);
- let return_types: Box<[_]> = return_idx_to_inside_id
+ let return_types: Vec<_> = return_idx_to_inside_id
.iter()
.map(|id| typing[id.idx()])
.chain(callee_succ_return_idx.map(|_| u32_ty))
.collect();
- let single_return = return_types.len() == 1;
let mut outlined = Function {
name: format!(
@@ -198,11 +197,7 @@ pub fn outline(
.map(|id| typing[id.idx()])
.chain(callee_pred_param_idx.map(|_| u32_ty))
.collect(),
- return_type: if single_return {
- return_types[0]
- } else {
- edit.add_type(Type::Product(return_types))
- },
+ return_types,
num_dynamic_constants: edit.get_num_dynamic_constant_params(),
entry: false,
nodes: vec![],
@@ -363,7 +358,6 @@ pub fn outline(
outlined.nodes.extend(select_top_phi_inputs);
// Add the return nodes.
- let cons_id = edit.add_zero_constant(outlined.return_type);
for ((exit, _), dom_return_values) in
zip(exit_points.iter(), exit_point_dom_return_values.iter())
{
@@ -398,29 +392,10 @@ pub fn outline(
data_ids.push(cons_node_id);
}
- // Build the return value
- let construct_id = if single_return {
- assert!(data_ids.len() == 1);
- data_ids.pop().unwrap()
- } else {
- let mut construct_id = NodeID::new(outlined.nodes.len());
- outlined.nodes.push(Node::Constant { id: cons_id });
- for (idx, data) in data_ids.into_iter().enumerate() {
- let write = Node::Write {
- collect: construct_id,
- data: data,
- indices: Box::new([Index::Field(idx)]),
- };
- construct_id = NodeID::new(outlined.nodes.len());
- outlined.nodes.push(write);
- }
- construct_id
- };
-
// Return the return product.
outlined.nodes.push(Node::Return {
control: convert_id(*exit),
- data: construct_id,
+ data: data_ids.into(),
});
}
@@ -515,29 +490,25 @@ pub fn outline(
(new_region_id, call_id)
};
- // Create the read nodes from the call node to get the outputs of the
- // outlined function (if there are multiple returned values)
- let output_reads: Vec<_> = if single_return {
- vec![call_id]
- } else {
- (0..return_idx_to_inside_id.len())
- .map(|idx| {
- let read = Node::Read {
- collect: call_id,
- indices: Box::new([Index::Field(idx)]),
- };
- edit.add_node(read)
- })
- .collect()
- };
- let indicator_read = callee_succ_return_idx.map(|idx| {
- let read = Node::Read {
- collect: call_id,
- indices: Box::new([Index::Field(idx)]),
+ // Create the data projection nodes from the call node to get the outputs of the outlined
+ // function
+ let output_projs: Vec<_> = (0..return_idx_to_inside_id.len())
+ .map(|idx| {
+ let proj = Node::DataProjection {
+ data: call_id,
+ selection: idx,
+ };
+ edit.add_node(proj)
+ })
+ .collect();
+ let indicator_proj = callee_succ_return_idx.map(|idx| {
+ let proj = Node::DataProjection {
+ data: call_id,
+ selection: idx,
};
- edit.add_node(read)
+ edit.add_node(proj)
});
- for (old_id, new_id) in zip(return_idx_to_inside_id.iter(), output_reads.iter()) {
+ for (old_id, new_id) in zip(return_idx_to_inside_id.iter(), output_projs.iter()) {
edit = edit.replace_all_uses(*old_id, *new_id)?;
}
@@ -554,18 +525,18 @@ pub fn outline(
});
let cmp_id = edit.add_node(Node::Binary {
op: BinaryOperator::EQ,
- left: indicator_read.unwrap(),
+ left: indicator_proj.unwrap(),
right: indicator_cons_node_id,
});
let if_id = edit.add_node(Node::If {
control: if_tree_acc,
cond: cmp_id,
});
- let false_id = edit.add_node(Node::Projection {
+ let false_id = edit.add_node(Node::ControlProjection {
control: if_id,
selection: 0,
});
- let true_id = edit.add_node(Node::Projection {
+ let true_id = edit.add_node(Node::ControlProjection {
control: if_id,
selection: 1,
});
diff --git a/hercules_opt/src/pred.rs b/hercules_opt/src/pred.rs
index 644c69d0df34d327c2c2e34bf8e0a915ddd68233..ed7c3a855b016608aa194cc9f2cd89f05d836bde 100644
--- a/hercules_opt/src/pred.rs
+++ b/hercules_opt/src/pred.rs
@@ -26,7 +26,7 @@ pub fn predication(editor: &mut FunctionEditor, typing: &Vec<TypeID>) {
// Look for two projections with the same branch.
let preds = preds.into_iter().filter_map(|id| {
nodes[id.idx()]
- .try_proj()
+ .try_control_proj()
.map(|(branch, selection)| (*id, branch, selection))
});
// Index projections by if branch.
diff --git a/hercules_opt/src/sroa.rs b/hercules_opt/src/sroa.rs
index 8865f863934233cb3675c57319c5e2746e339aaf..eff0a7296161149b32def1da5e6b46d681fc9434 100644
--- a/hercules_opt/src/sroa.rs
+++ b/hercules_opt/src/sroa.rs
@@ -27,9 +27,10 @@ use crate::*;
* are broken up into ternary nodes for the individual fields
*
* - Call: the call node can use a product value as an argument to another
- * function, and can produce a product value as a result. Argument values
- * will be constructed at the call site and the return value will be broken
- * into individual fields
+ * function, argument values will be constructed at the call site
+ *
+ * - DataProjection: data projection nodes can produce a product value that was
+ * returned by a function, we will break the value into individual fields
*
* - Read: the read node reads primitive fields from product values - these get
* replaced by a direct use of the field value
@@ -71,8 +72,9 @@ pub fn sroa(
// First: determine all nodes which interact with products (as described above)
let mut product_nodes: Vec<NodeID> = vec![];
- // We track call and return nodes separately since they (may) require constructing new products
- // for the call's arguments or the return's value
+ // We track call, data projection, and return nodes separately since they (may) require
+ // constructing new products for the call's arguments, data projection's value, or a
+ // returned value
let mut call_return_nodes: Vec<NodeID> = vec![];
for node in reverse_postorder {
@@ -303,37 +305,57 @@ pub fn sroa(
}
}
- // We add all calls to the call/return list and check their arguments later
- Node::Call { .. } => call_return_nodes.push(*node),
- Node::Return { control: _, data } if can_sroa_type(editor, types[&data]) => {
- call_return_nodes.push(*node)
+ // We add all calls and returns to the call/return list and check their
+ // arguments/return values later
+ Node::Call { .. } | Node::Return { .. } => call_return_nodes.push(*node),
+ // We add DataProjetion nodes that produce SROAable values
+ Node::DataProjection { .. } if can_sroa_type(editor, types[&node]) => {
+ call_return_nodes.push(*node);
}
_ => (),
}
}
- // Next, we handle calls and returns. For returns, we will insert nodes that read each field of
- // the returned product and then write them into a new product. These writes are not put into
- // the list of product nodes since they must remain but the reads are so that they will be
- // replaced later on.
- // For calls, we do a similar process for each (product) argument. Additionally, if the call
- // returns a product, we create reads for each field in that product and store it into our
- // field map
+ // Next, we handle calls and returns. For returns, for each returned value that is a product,
+ // we will insert nodes that read each field of it and then write them into a new product.
+ // The writes we create are not put into the list of product nodes since they must remain but
+ // the reads are put in the list so that they will be replaced later on.
+ // For calls, we do a similar process for each (product) argument.
+ // For data projection that produce product values, we create reads for each field of that
+ // product and store it into our field map
for node in call_return_nodes {
match &editor.func().nodes[node.idx()] {
Node::Return { control, data } => {
- assert!(can_sroa_type(editor, types[&data]));
let control = *control;
- let new_data = reconstruct_product(editor, types[&data], *data, &mut product_nodes);
- editor.edit(|mut edit| {
- let new_return = edit.add_node(Node::Return {
- control,
- data: new_data,
+ let data = data.clone();
+
+ let (new_data, changed) =
+ data.into_iter()
+ .fold((vec![], false), |(mut vals, changed), val_id| {
+ if !can_sroa_type(editor, types[val_id]) {
+ vals.push(*val_id);
+ (vals, changed)
+ } else {
+ vals.push(reconstruct_product(
+ editor,
+ types[val_id],
+ *val_id,
+ &mut product_nodes,
+ ));
+ (vals, true)
+ }
+ });
+ if changed {
+ editor.edit(|mut edit| {
+ let new_return = edit.add_node(Node::Return {
+ control,
+ data: new_data.into(),
+ });
+ edit.sub_edit(node, new_return);
+ edit.delete_node(node)
});
- edit.sub_edit(node, new_return);
- edit.delete_node(node)
- });
+ }
}
Node::Call {
control,
@@ -346,53 +368,42 @@ pub fn sroa(
let dynamic_constants = dynamic_constants.clone();
let args = args.clone();
- // If the call returns a product that we can sroa, we generate reads for each field
- let fields = if can_sroa_type(editor, types[&node]) {
- Some(generate_reads(editor, types[&node], node))
- } else {
- None
- };
+ let (new_args, changed) =
+ args.into_iter()
+ .fold((vec![], false), |(mut vals, changed), arg| {
+ if !can_sroa_type(editor, types[arg]) {
+ vals.push(*arg);
+ (vals, changed)
+ } else {
+ vals.push(reconstruct_product(
+ editor,
+ types[arg],
+ *arg,
+ &mut product_nodes,
+ ));
+ (vals, true)
+ }
+ });
- let mut new_args = vec![];
- for arg in args {
- if can_sroa_type(editor, types[&arg]) {
- new_args.push(reconstruct_product(
- editor,
- types[&arg],
- arg,
- &mut product_nodes,
- ));
- } else {
- new_args.push(arg);
- }
- }
- editor.edit(|mut edit| {
- let new_call = edit.add_node(Node::Call {
- control,
- function,
- dynamic_constants,
- args: new_args.into(),
- });
- edit.sub_edit(node, new_call);
- let edit = edit.replace_all_uses(node, new_call)?;
- let edit = edit.delete_node(node)?;
-
- // Since we've replaced uses of calls with the new node, we update the type
- // information so that we can retrieve the type of the new call if needed
- // Because the other nodes we've created so far are only used in very
- // particular ways (i.e. are not used by arbitrary nodes) we don't need their
- // type information but do for the new calls
- types.insert(new_call, types[&node]);
-
- match fields {
- None => {}
- Some(fields) => {
- field_map.insert(new_call, fields);
- }
- }
+ if changed {
+ editor.edit(|mut edit| {
+ let new_call = edit.add_node(Node::Call {
+ control,
+ function,
+ dynamic_constants,
+ args: new_args.into(),
+ });
+ edit.sub_edit(node, new_call);
+ let edit = edit.replace_all_uses(node, new_call)?;
+ let edit = edit.delete_node(node)?;
- Ok(edit)
- });
+ Ok(edit)
+ });
+ }
+ }
+ Node::DataProjection { .. } => {
+ assert!(can_sroa_type(editor, types[&node]));
+ field_map.insert(node, generate_reads(editor, types[&node], node));
}
_ => panic!("Processing non-call or return node"),
}
@@ -1055,6 +1066,7 @@ fn generate_constant(editor: &mut FunctionEditor, typ: TypeID) -> ConstantID {
add_const!(editor, Constant::Array(typ))
}
Type::Control => panic!("Cannot create constant of control type"),
+ Type::MultiReturn(_) => panic!("Cannot create constant of multi-return type"),
}
}
diff --git a/hercules_opt/src/unforkify.rs b/hercules_opt/src/unforkify.rs
index b44ed8df82b494b7da0aff006587246c501f8e5d..2d6cf7b36f44864d67ef4ee505203de12ee59bff 100644
--- a/hercules_opt/src/unforkify.rs
+++ b/hercules_opt/src/unforkify.rs
@@ -205,11 +205,11 @@ pub fn unforkify(
control: fork_control,
cond: guard_cond_id,
};
- let guard_taken_proj = Node::Projection {
+ let guard_taken_proj = Node::ControlProjection {
control: guard_if_id,
selection: 1,
};
- let guard_skipped_proj = Node::Projection {
+ let guard_skipped_proj = Node::ControlProjection {
control: guard_if_id,
selection: 0,
};
@@ -224,11 +224,11 @@ pub fn unforkify(
control: join_control,
cond: neq_id,
};
- let proj_back = Node::Projection {
+ let proj_back = Node::ControlProjection {
control: if_id,
selection: 1,
};
- let proj_exit = Node::Projection {
+ let proj_exit = Node::ControlProjection {
control: if_id,
selection: 0,
};
diff --git a/hercules_opt/src/utils.rs b/hercules_opt/src/utils.rs
index 1806d5c740e57a666f98ebf6c0ba40ee9a6461bd..c165c0a0f29537a696510d92368b9624ddd68866 100644
--- a/hercules_opt/src/utils.rs
+++ b/hercules_opt/src/utils.rs
@@ -244,31 +244,42 @@ pub fn collapse_returns(editor: &mut FunctionEditor) -> Option<NodeID> {
if returns.len() == 1 {
return Some(returns[0]);
}
- let preds_before_returns: Vec<NodeID> = returns
+ let preds_before_returns: Box<[NodeID]> = returns
.iter()
.map(|ret_id| get_uses(&editor.func().nodes[ret_id.idx()]).as_ref()[0])
.collect();
- let data_to_return: Vec<NodeID> = returns
- .iter()
- .map(|ret_id| get_uses(&editor.func().nodes[ret_id.idx()]).as_ref()[1])
+
+ let num_return_data = editor.func().return_types.len();
+ let data_to_return: Vec<Box<[NodeID]>> = (0..num_return_data)
+ .map(|idx| {
+ returns
+ .iter()
+ .map(|ret_id| get_uses(&editor.func().nodes[ret_id.idx()]).as_ref()[idx + 1])
+ .collect()
+ })
.collect();
// All of the old returns get replaced in a single edit.
let mut new_return = None;
editor.edit(|mut edit| {
let region = edit.add_node(Node::Region {
- preds: preds_before_returns.into_boxed_slice(),
- });
- let phi = edit.add_node(Node::Phi {
- control: region,
- data: data_to_return.into_boxed_slice(),
+ preds: preds_before_returns,
});
+ let return_vals = data_to_return
+ .into_iter()
+ .map(|data| {
+ edit.add_node(Node::Phi {
+ control: region,
+ data,
+ })
+ })
+ .collect();
for ret in returns {
edit = edit.delete_node(ret)?;
}
new_return = Some(edit.add_node(Node::Return {
control: region,
- data: phi,
+ data: return_vals,
}));
Ok(edit)
});
@@ -293,10 +304,11 @@ pub fn ensure_between_control_flow(editor: &mut FunctionEditor) -> Option<NodeID
.filter(|id| editor.func().nodes[id.idx()].is_control())
.next()
.unwrap();
- let Node::Return { control, data } = editor.func().nodes[ret.idx()] else {
+ let Node::Return { control, ref data } = editor.func().nodes[ret.idx()] else {
panic!("PANIC: A Hercules function with only two control nodes must have a return node be the other control node, other than the start node.")
};
assert_eq!(control, NodeID::new(0), "PANIC: The only other control node in a Hercules function, the return node, is not using the start node.");
+ let data = data.clone();
let mut region_id = None;
editor.edit(|mut edit| {
edit = edit.delete_node(ret)?;