diff --git a/src/compiler/nir/meson.build b/src/compiler/nir/meson.build index 8ecba3b3d1f..8fb99efc4bd 100644 --- a/src/compiler/nir/meson.build +++ b/src/compiler/nir/meson.build @@ -310,6 +310,7 @@ files_libnir = files( '../spirv/vtn_alu.c', '../spirv/vtn_amd.c', '../spirv/vtn_cfg.c', + '../spirv/vtn_structured_cfg.c', '../spirv/vtn_glsl450.c', '../spirv/vtn_opencl.c', '../spirv/vtn_private.h', diff --git a/src/compiler/spirv/spirv_to_nir.c b/src/compiler/spirv/spirv_to_nir.c index 9f2e91b5a3b..5997e861f3b 100644 --- a/src/compiler/spirv/spirv_to_nir.c +++ b/src/compiler/spirv/spirv_to_nir.c @@ -43,6 +43,8 @@ uint32_t mesa_spirv_debug = 0; static const struct debug_named_value mesa_spirv_debug_control[] = { + { "structured", MESA_SPIRV_DEBUG_STRUCTURED, + "Print information of the SPIR-V structured control flow parsing" }, DEBUG_NAMED_VALUE_END, }; @@ -6694,8 +6696,7 @@ spirv_to_nir(const uint32_t *words, size_t word_count, bool progress; do { progress = false; - vtn_foreach_cf_node(node, &b->functions) { - struct vtn_function *func = vtn_cf_node_as_function(node); + vtn_foreach_function(func, &b->functions) { if ((options->create_library || func->referenced) && !func->emitted) { b->const_table = _mesa_pointer_hash_table_create(b); @@ -6719,6 +6720,9 @@ spirv_to_nir(const uint32_t *words, size_t word_count, /* structurize the CFG */ nir_lower_goto_ifs(b->shader); + + nir_validate_shader(b->shader, "after spirv cfg"); + nir_lower_continue_constructs(b->shader); /* A SPIR-V module can have multiple shaders stages and also multiple diff --git a/src/compiler/spirv/tests/control_flow_tests.cpp b/src/compiler/spirv/tests/control_flow_tests.cpp index 04b8318ddaf..c6ca67ae681 100644 --- a/src/compiler/spirv/tests/control_flow_tests.cpp +++ b/src/compiler/spirv/tests/control_flow_tests.cpp @@ -137,7 +137,7 @@ OpFunctionEnd ASSERT_TRUE(shader); } -TEST_F(ControlFlow, DISABLED_EarlyMerge) +TEST_F(ControlFlow, EarlyMerge) { // From https://gitlab.khronos.org/spirv/SPIR-V/-/issues/640. diff --git a/src/compiler/spirv/vtn_cfg.c b/src/compiler/spirv/vtn_cfg.c index b02c11b8d7c..4182a1a600f 100644 --- a/src/compiler/spirv/vtn_cfg.c +++ b/src/compiler/spirv/vtn_cfg.c @@ -26,12 +26,6 @@ #include "nir/nir_vla.h" #include "util/u_debug.h" -static struct vtn_block * -vtn_block(struct vtn_builder *b, uint32_t value_id) -{ - return vtn_value(b, value_id, vtn_value_type_block)->block; -} - static unsigned glsl_type_count_function_params(const struct glsl_type *type) { @@ -171,7 +165,7 @@ function_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member, } } -static bool +bool vtn_cfg_handle_prepass_instruction(struct vtn_builder *b, SpvOp opcode, const uint32_t *w, unsigned count) { @@ -180,11 +174,10 @@ vtn_cfg_handle_prepass_instruction(struct vtn_builder *b, SpvOp opcode, vtn_assert(b->func == NULL); b->func = rzalloc(b, struct vtn_function); - b->func->node.type = vtn_cf_node_type_function; - b->func->node.parent = NULL; list_inithead(&b->func->body); b->func->linkage = SpvLinkageTypeMax; b->func->control = w[3]; + list_inithead(&b->func->constructs); UNUSED const struct glsl_type *result_type = vtn_get_type(b, w[1])->type; struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_function); @@ -276,17 +269,18 @@ vtn_cfg_handle_prepass_instruction(struct vtn_builder *b, SpvOp opcode, case SpvOpLabel: { vtn_assert(b->block == NULL); b->block = rzalloc(b, struct vtn_block); - b->block->node.type = vtn_cf_node_type_block; b->block->label = w; vtn_push_value(b, w[1], vtn_value_type_block)->block = b->block; + b->func->block_count++; + if (b->func->start_block == NULL) { /* This is the first block encountered for this function. In this * case, we set the start block and add it to the list of * implemented functions that we'll walk later. */ b->func->start_block = b->block; - list_addtail(&b->func->node.link, &b->functions); + list_addtail(&b->func->link, &b->functions); } break; } @@ -327,217 +321,9 @@ vtn_cfg_handle_prepass_instruction(struct vtn_builder *b, SpvOp opcode, return true; } -/* This function performs a depth-first search of the cases and puts them - * in fall-through order. - */ -static void -vtn_order_case(struct vtn_switch *swtch, struct vtn_case *cse) -{ - if (cse->visited) - return; - - cse->visited = true; - - list_del(&cse->node.link); - - if (cse->fallthrough) { - vtn_order_case(swtch, cse->fallthrough); - - /* If we have a fall-through, place this case right before the case it - * falls through to. This ensures that fallthroughs come one after - * the other. These two can never get separated because that would - * imply something else falling through to the same case. Also, this - * can't break ordering because the DFS ensures that this case is - * visited before anything that falls through to it. - */ - list_addtail(&cse->node.link, &cse->fallthrough->node.link); - } else { - list_add(&cse->node.link, &swtch->cases); - } -} - -static void -vtn_switch_order_cases(struct vtn_switch *swtch) -{ - struct list_head cases; - list_replace(&swtch->cases, &cases); - list_inithead(&swtch->cases); - while (!list_is_empty(&cases)) { - struct vtn_case *cse = - list_first_entry(&cases, struct vtn_case, node.link); - vtn_order_case(swtch, cse); - } -} - -static void -vtn_block_set_merge_cf_node(struct vtn_builder *b, struct vtn_block *block, - struct vtn_cf_node *cf_node) -{ - vtn_fail_if(block->merge_cf_node != NULL, - "The merge block declared by a header block cannot be a " - "merge block declared by any other header block."); - - block->merge_cf_node = cf_node; -} - -#define VTN_DECL_CF_NODE_FIND(_type) \ -static inline struct vtn_##_type * \ -vtn_cf_node_find_##_type(struct vtn_cf_node *node) \ -{ \ - while (node && node->type != vtn_cf_node_type_##_type) \ - node = node->parent; \ - return (struct vtn_##_type *)node; \ -} - -UNUSED VTN_DECL_CF_NODE_FIND(if) -VTN_DECL_CF_NODE_FIND(loop) -VTN_DECL_CF_NODE_FIND(case) -VTN_DECL_CF_NODE_FIND(switch) -VTN_DECL_CF_NODE_FIND(function) - -static enum vtn_branch_type -vtn_handle_branch(struct vtn_builder *b, - struct vtn_cf_node *cf_parent, - struct vtn_block *target_block) -{ - struct vtn_loop *loop = vtn_cf_node_find_loop(cf_parent); - - /* Detect a loop back-edge first. That way none of the code below - * accidentally operates on a loop back-edge. - */ - if (loop && target_block == loop->header_block) - return vtn_branch_type_loop_back_edge; - - /* Try to detect fall-through */ - if (target_block->switch_case) { - /* When it comes to handling switch cases, we can break calls to - * vtn_handle_branch into two cases: calls from within a case construct - * and calls for the jump to each case construct. In the second case, - * cf_parent is the vtn_switch itself and vtn_cf_node_find_case() will - * return the outer switch case in which this switch is contained. It's - * fine if the target block is a switch case from an outer switch as - * long as it is also the switch break for this switch. - */ - struct vtn_case *switch_case = vtn_cf_node_find_case(cf_parent); - - /* This doesn't get called for the OpSwitch */ - vtn_fail_if(switch_case == NULL, - "A switch case can only be entered through an OpSwitch or " - "falling through from another switch case."); - - /* Because block->switch_case is only set on the entry block for a given - * switch case, we only ever get here if we're jumping to the start of a - * switch case. It's possible, however, that a switch case could jump - * to itself via a back-edge. That *should* get caught by the loop - * handling case above but if we have a back edge without a loop merge, - * we could en up here. - */ - vtn_fail_if(target_block->switch_case == switch_case, - "A switch cannot fall-through to itself. Likely, there is " - "a back-edge which is not to a loop header."); - - vtn_fail_if(target_block->switch_case->node.parent != - switch_case->node.parent, - "A switch case fall-through must come from the same " - "OpSwitch construct"); - - vtn_fail_if(switch_case->fallthrough != NULL && - switch_case->fallthrough != target_block->switch_case, - "Each case construct can have at most one branch to " - "another case construct"); - - switch_case->fallthrough = target_block->switch_case; - - /* We don't immediately return vtn_branch_type_switch_fallthrough - * because it may also be a loop or switch break for an inner loop or - * switch and that takes precedence. - */ - } - - if (loop && target_block == loop->cont_block) - return vtn_branch_type_loop_continue; - - /* We walk blocks as a breadth-first search on the control-flow construct - * tree where, when we find a construct, we add the vtn_cf_node for that - * construct and continue iterating at the merge target block (if any). - * Therefore, we want merges whose with parent == cf_parent to be treated - * as regular branches. We only want to consider merges if they break out - * of the current CF construct. - */ - if (target_block->merge_cf_node != NULL && - target_block->merge_cf_node->parent != cf_parent) { - switch (target_block->merge_cf_node->type) { - case vtn_cf_node_type_if: - for (struct vtn_cf_node *node = cf_parent; - node != target_block->merge_cf_node; node = node->parent) { - vtn_fail_if(node == NULL || node->type != vtn_cf_node_type_if, - "Branching to the merge block of a selection " - "construct can only be used to break out of a " - "selection construct"); - - struct vtn_if *if_stmt = vtn_cf_node_as_if(node); - - /* This should be guaranteed by our iteration */ - assert(if_stmt->merge_block != target_block); - - vtn_fail_if(if_stmt->merge_block != NULL, - "Branching to the merge block of a selection " - "construct can only be used to break out of the " - "inner most nested selection level"); - } - return vtn_branch_type_if_merge; - - case vtn_cf_node_type_loop: - vtn_fail_if(target_block->merge_cf_node != &loop->node, - "Loop breaks can only break out of the inner most " - "nested loop level"); - return vtn_branch_type_loop_break; - - case vtn_cf_node_type_switch: { - struct vtn_switch *swtch = vtn_cf_node_find_switch(cf_parent); - vtn_fail_if(target_block->merge_cf_node != &swtch->node, - "Switch breaks can only break out of the inner most " - "nested switch level"); - return vtn_branch_type_switch_break; - } - - default: - unreachable("Invalid CF node type for a merge"); - } - } - - if (target_block->switch_case) - return vtn_branch_type_switch_fallthrough; - - return vtn_branch_type_none; -} - -struct vtn_cfg_work_item { - struct list_head link; - - struct vtn_cf_node *cf_parent; - struct list_head *cf_list; - struct vtn_block *start_block; -}; - -static void -vtn_add_cfg_work_item(struct vtn_builder *b, - struct list_head *work_list, - struct vtn_cf_node *cf_parent, - struct list_head *cf_list, - struct vtn_block *start_block) -{ - struct vtn_cfg_work_item *work = ralloc(b, struct vtn_cfg_work_item); - work->cf_parent = cf_parent; - work->cf_list = cf_list; - work->start_block = start_block; - list_addtail(&work->link, work_list); -} - /* returns the default block */ -static void +void vtn_parse_switch(struct vtn_builder *b, - struct vtn_switch *swtch, const uint32_t *branch, struct list_head *case_list) { @@ -579,14 +365,11 @@ vtn_parse_switch(struct vtn_builder *b, cse = case_entry->data; } else { cse = rzalloc(b, struct vtn_case); - - cse->node.type = vtn_cf_node_type_case; - cse->node.parent = swtch ? &swtch->node : NULL; cse->block = case_block; - list_inithead(&cse->body); + cse->block->switch_case = cse; util_dynarray_init(&cse->values, b); - list_addtail(&cse->node.link, case_list); + list_addtail(&cse->link, case_list); _mesa_hash_table_insert(block_to_case, case_block, cse); } @@ -602,278 +385,6 @@ vtn_parse_switch(struct vtn_builder *b, _mesa_hash_table_destroy(block_to_case, NULL); } -/* Processes a block and returns the next block to process or NULL if we've - * reached the end of the construct. - */ -static struct vtn_block * -vtn_process_block(struct vtn_builder *b, - struct list_head *work_list, - struct vtn_cf_node *cf_parent, - struct list_head *cf_list, - struct vtn_block *block) -{ - if (!list_is_empty(cf_list)) { - /* vtn_process_block() acts like an iterator: it processes the given - * block and then returns the next block to process. For a given - * control-flow construct, vtn_build_cfg() calls vtn_process_block() - * repeatedly until it finally returns NULL. Therefore, we know that - * the only blocks on which vtn_process_block() can be called are either - * the first block in a construct or a block that vtn_process_block() - * returned for the current construct. If cf_list is empty then we know - * that we're processing the first block in the construct and we have to - * add it to the list. - * - * If cf_list is not empty, then it must be the block returned by the - * previous call to vtn_process_block(). We know a priori that - * vtn_process_block only returns either normal branches - * (vtn_branch_type_none) or merge target blocks. - */ - switch (vtn_handle_branch(b, cf_parent, block)) { - case vtn_branch_type_none: - /* For normal branches, we want to process them and add them to the - * current construct. Merge target blocks also look like normal - * branches from the perspective of this construct. See also - * vtn_handle_branch(). - */ - break; - - case vtn_branch_type_loop_continue: - case vtn_branch_type_switch_fallthrough: - /* The two cases where we can get early exits from a construct that - * are not to that construct's merge target are loop continues and - * switch fall-throughs. In these cases, we need to break out of the - * current construct by returning NULL. - */ - return NULL; - - default: - /* The only way we can get here is if something was used as two kinds - * of merges at the same time and that's illegal. - */ - vtn_fail("A block was used as a merge target from two or more " - "structured control-flow constructs"); - } - } - - /* Once a block has been processed, it is placed into and the list link - * will point to something non-null. If we see a node we've already - * processed here, it either exists in multiple functions or it's an - * invalid back-edge. - */ - if (block->node.parent != NULL) { - vtn_fail_if(vtn_cf_node_find_function(&block->node) != - vtn_cf_node_find_function(cf_parent), - "A block cannot exist in two functions at the " - "same time"); - - vtn_fail("Invalid back or cross-edge in the CFG"); - } - - if (block->merge && (*block->merge & SpvOpCodeMask) == SpvOpLoopMerge && - block->loop == NULL) { - vtn_fail_if((*block->branch & SpvOpCodeMask) != SpvOpBranch && - (*block->branch & SpvOpCodeMask) != SpvOpBranchConditional, - "An OpLoopMerge instruction must immediately precede " - "either an OpBranch or OpBranchConditional instruction."); - - struct vtn_loop *loop = rzalloc(b, struct vtn_loop); - - loop->node.type = vtn_cf_node_type_loop; - loop->node.parent = cf_parent; - list_inithead(&loop->body); - list_inithead(&loop->cont_body); - loop->header_block = block; - loop->break_block = vtn_block(b, block->merge[1]); - loop->cont_block = vtn_block(b, block->merge[2]); - loop->control = block->merge[3]; - - list_addtail(&loop->node.link, cf_list); - block->loop = loop; - - /* Note: The work item for the main loop body will start with the - * current block as its start block. If we weren't careful, we would - * get here again and end up in an infinite loop. This is why we set - * block->loop above and check for it before creating one. This way, - * we only create the loop once and the second iteration that tries to - * handle this loop goes to the cases below and gets handled as a - * regular block. - */ - vtn_add_cfg_work_item(b, work_list, &loop->node, - &loop->body, loop->header_block); - - /* For continue targets, SPIR-V guarantees the following: - * - * - the Continue Target must dominate the back-edge block - * - the back-edge block must post dominate the Continue Target - * - * If the header block is the same as the continue target, this - * condition is trivially satisfied and there is no real continue - * section. - */ - if (loop->cont_block != loop->header_block) { - vtn_add_cfg_work_item(b, work_list, &loop->node, - &loop->cont_body, loop->cont_block); - } - - vtn_block_set_merge_cf_node(b, loop->break_block, &loop->node); - - return loop->break_block; - } - - /* Add the block to the CF list */ - block->node.parent = cf_parent; - list_addtail(&block->node.link, cf_list); - - switch (*block->branch & SpvOpCodeMask) { - case SpvOpBranch: { - struct vtn_block *branch_block = vtn_block(b, block->branch[1]); - - block->branch_type = vtn_handle_branch(b, cf_parent, branch_block); - - if (block->branch_type == vtn_branch_type_none) - return branch_block; - else - return NULL; - } - - case SpvOpReturn: - case SpvOpReturnValue: - block->branch_type = vtn_branch_type_return; - return NULL; - - case SpvOpKill: - block->branch_type = vtn_branch_type_discard; - return NULL; - - case SpvOpTerminateInvocation: - block->branch_type = vtn_branch_type_terminate_invocation; - return NULL; - - case SpvOpIgnoreIntersectionKHR: - block->branch_type = vtn_branch_type_ignore_intersection; - return NULL; - - case SpvOpTerminateRayKHR: - block->branch_type = vtn_branch_type_terminate_ray; - return NULL; - - case SpvOpEmitMeshTasksEXT: - block->branch_type = vtn_branch_type_emit_mesh_tasks; - return NULL; - - case SpvOpBranchConditional: { - struct vtn_value *cond_val = vtn_untyped_value(b, block->branch[1]); - vtn_fail_if(!cond_val->type || - cond_val->type->base_type != vtn_base_type_scalar || - cond_val->type->type != glsl_bool_type(), - "Condition must be a Boolean type scalar"); - - struct vtn_if *if_stmt = rzalloc(b, struct vtn_if); - - if_stmt->node.type = vtn_cf_node_type_if; - if_stmt->node.parent = cf_parent; - if_stmt->header_block = block; - list_inithead(&if_stmt->then_body); - list_inithead(&if_stmt->else_body); - - list_addtail(&if_stmt->node.link, cf_list); - - if (block->merge && - (*block->merge & SpvOpCodeMask) == SpvOpSelectionMerge) { - /* We may not always have a merge block and that merge doesn't - * technically have to be an OpSelectionMerge. We could have a block - * with an OpLoopMerge which ends in an OpBranchConditional. - */ - if_stmt->merge_block = vtn_block(b, block->merge[1]); - vtn_block_set_merge_cf_node(b, if_stmt->merge_block, &if_stmt->node); - - if_stmt->control = block->merge[2]; - } - - struct vtn_block *then_block = vtn_block(b, block->branch[2]); - if_stmt->then_type = vtn_handle_branch(b, &if_stmt->node, then_block); - if (if_stmt->then_type == vtn_branch_type_none) { - vtn_add_cfg_work_item(b, work_list, &if_stmt->node, - &if_stmt->then_body, then_block); - } - - struct vtn_block *else_block = vtn_block(b, block->branch[3]); - if (then_block != else_block) { - if_stmt->else_type = vtn_handle_branch(b, &if_stmt->node, else_block); - if (if_stmt->else_type == vtn_branch_type_none) { - vtn_add_cfg_work_item(b, work_list, &if_stmt->node, - &if_stmt->else_body, else_block); - } - } - - return if_stmt->merge_block; - } - - case SpvOpSwitch: { - struct vtn_switch *swtch = rzalloc(b, struct vtn_switch); - - swtch->node.type = vtn_cf_node_type_switch; - swtch->node.parent = cf_parent; - swtch->selector = block->branch[1]; - list_inithead(&swtch->cases); - - list_addtail(&swtch->node.link, cf_list); - - /* We may not always have a merge block */ - if (block->merge) { - vtn_fail_if((*block->merge & SpvOpCodeMask) != SpvOpSelectionMerge, - "An OpLoopMerge instruction must immediately precede " - "either an OpBranch or OpBranchConditional " - "instruction."); - swtch->break_block = vtn_block(b, block->merge[1]); - vtn_block_set_merge_cf_node(b, swtch->break_block, &swtch->node); - } - - /* First, we go through and record all of the cases. */ - vtn_parse_switch(b, swtch, block->branch, &swtch->cases); - - /* Gather the branch types for the switch */ - vtn_foreach_cf_node(case_node, &swtch->cases) { - struct vtn_case *cse = vtn_cf_node_as_case(case_node); - - cse->type = vtn_handle_branch(b, &swtch->node, cse->block); - switch (cse->type) { - case vtn_branch_type_none: - /* This is a "real" cases which has stuff in it */ - vtn_fail_if(cse->block->switch_case != NULL, - "OpSwitch has a case which is also in another " - "OpSwitch construct"); - cse->block->switch_case = cse; - vtn_add_cfg_work_item(b, work_list, &cse->node, - &cse->body, cse->block); - break; - - case vtn_branch_type_switch_break: - case vtn_branch_type_loop_break: - case vtn_branch_type_loop_continue: - /* Switch breaks as well as loop breaks and continues can be - * used to break out of a switch construct or as direct targets - * of the OpSwitch. - */ - break; - - default: - vtn_fail("Target of OpSwitch is not a valid structured exit " - "from the switch construct."); - } - } - - return swtch->break_block; - } - - case SpvOpUnreachable: - return NULL; - - default: - vtn_fail("Block did not end with a valid branch instruction"); - } -} - void vtn_build_cfg(struct vtn_builder *b, const uint32_t *words, const uint32_t *end) { @@ -883,34 +394,10 @@ vtn_build_cfg(struct vtn_builder *b, const uint32_t *words, const uint32_t *end) if (b->shader->info.stage == MESA_SHADER_KERNEL) return; - vtn_foreach_cf_node(func_node, &b->functions) { - struct vtn_function *func = vtn_cf_node_as_function(func_node); - - /* We build the CFG for each function by doing a breadth-first search on - * the control-flow graph. We keep track of our state using a worklist. - * Doing a BFS ensures that we visit each structured control-flow - * construct and its merge node before we visit the stuff inside the - * construct. - */ - struct list_head work_list; - list_inithead(&work_list); - vtn_add_cfg_work_item(b, &work_list, &func->node, &func->body, - func->start_block); - - while (!list_is_empty(&work_list)) { - struct vtn_cfg_work_item *work = - list_first_entry(&work_list, struct vtn_cfg_work_item, link); - list_del(&work->link); - - for (struct vtn_block *block = work->start_block; block; ) { - block = vtn_process_block(b, &work_list, work->cf_parent, - work->cf_list, block); - } - } - } + vtn_build_structured_cfg(b, words, end); } -static bool +bool vtn_handle_phis_first_pass(struct vtn_builder *b, SpvOp opcode, const uint32_t *w, unsigned count) { @@ -984,7 +471,7 @@ vtn_handle_phi_second_pass(struct vtn_builder *b, SpvOp opcode, return true; } -static void +void vtn_emit_ret_store(struct vtn_builder *b, const struct vtn_block *block) { if ((*block->branch & SpvOpCodeMask) != SpvOpReturnValue) @@ -1001,299 +488,6 @@ vtn_emit_ret_store(struct vtn_builder *b, const struct vtn_block *block) vtn_local_store(b, src, ret_deref, 0); } -static void -vtn_emit_branch(struct vtn_builder *b, enum vtn_branch_type branch_type, - const struct vtn_block *block, - nir_variable *switch_fall_var, bool *has_switch_break) -{ - switch (branch_type) { - case vtn_branch_type_if_merge: - break; /* Nothing to do */ - case vtn_branch_type_switch_break: - nir_store_var(&b->nb, switch_fall_var, nir_imm_false(&b->nb), 1); - *has_switch_break = true; - break; - case vtn_branch_type_switch_fallthrough: - break; /* Nothing to do */ - case vtn_branch_type_loop_break: - nir_jump(&b->nb, nir_jump_break); - break; - case vtn_branch_type_loop_continue: - nir_jump(&b->nb, nir_jump_continue); - break; - case vtn_branch_type_loop_back_edge: - break; - case vtn_branch_type_return: - vtn_assert(block); - vtn_emit_ret_store(b, block); - nir_jump(&b->nb, nir_jump_return); - break; - case vtn_branch_type_discard: - if (b->convert_discard_to_demote) - nir_demote(&b->nb); - else - nir_discard(&b->nb); - break; - case vtn_branch_type_terminate_invocation: - nir_terminate(&b->nb); - break; - case vtn_branch_type_ignore_intersection: - nir_ignore_ray_intersection(&b->nb); - nir_jump(&b->nb, nir_jump_halt); - break; - case vtn_branch_type_terminate_ray: - nir_terminate_ray(&b->nb); - nir_jump(&b->nb, nir_jump_halt); - break; - case vtn_branch_type_emit_mesh_tasks: { - assert(block); - assert(block->branch); - - const uint32_t *w = block->branch; - vtn_assert((w[0] & SpvOpCodeMask) == SpvOpEmitMeshTasksEXT); - - /* Launches mesh shader workgroups from the task shader. - * Arguments are: vec(x, y, z), payload pointer - */ - nir_ssa_def *dimensions = - nir_vec3(&b->nb, vtn_get_nir_ssa(b, w[1]), - vtn_get_nir_ssa(b, w[2]), - vtn_get_nir_ssa(b, w[3])); - - /* The payload variable is optional. - * We don't have a NULL deref in NIR, so just emit the explicit - * intrinsic when there is no payload. - */ - const unsigned count = w[0] >> SpvWordCountShift; - if (count == 4) - nir_launch_mesh_workgroups(&b->nb, dimensions); - else if (count == 5) - nir_launch_mesh_workgroups_with_payload_deref(&b->nb, dimensions, - vtn_get_nir_ssa(b, w[4])); - else - vtn_fail("Invalid EmitMeshTasksEXT."); - - nir_jump(&b->nb, nir_jump_halt); - break; - } - default: - vtn_fail("Invalid branch type"); - } -} - -static nir_ssa_def * -vtn_switch_case_condition(struct vtn_builder *b, struct vtn_switch *swtch, - nir_ssa_def *sel, struct vtn_case *cse) -{ - if (cse->is_default) { - nir_ssa_def *any = nir_imm_false(&b->nb); - vtn_foreach_cf_node(other_node, &swtch->cases) { - struct vtn_case *other = vtn_cf_node_as_case(other_node); - if (other->is_default) - continue; - - any = nir_ior(&b->nb, any, - vtn_switch_case_condition(b, swtch, sel, other)); - } - return nir_inot(&b->nb, any); - } else { - nir_ssa_def *cond = nir_imm_false(&b->nb); - util_dynarray_foreach(&cse->values, uint64_t, val) - cond = nir_ior(&b->nb, cond, nir_ieq_imm(&b->nb, sel, *val)); - return cond; - } -} - -static nir_loop_control -vtn_loop_control(struct vtn_builder *b, struct vtn_loop *vtn_loop) -{ - if (vtn_loop->control == SpvLoopControlMaskNone) - return nir_loop_control_none; - else if (vtn_loop->control & SpvLoopControlDontUnrollMask) - return nir_loop_control_dont_unroll; - else if (vtn_loop->control & SpvLoopControlUnrollMask) - return nir_loop_control_unroll; - else if (vtn_loop->control & SpvLoopControlDependencyInfiniteMask || - vtn_loop->control & SpvLoopControlDependencyLengthMask || - vtn_loop->control & SpvLoopControlMinIterationsMask || - vtn_loop->control & SpvLoopControlMaxIterationsMask || - vtn_loop->control & SpvLoopControlIterationMultipleMask || - vtn_loop->control & SpvLoopControlPeelCountMask || - vtn_loop->control & SpvLoopControlPartialCountMask) { - /* We do not do anything special with these yet. */ - return nir_loop_control_none; - } else { - vtn_fail("Invalid loop control"); - } -} - -static nir_selection_control -vtn_selection_control(struct vtn_builder *b, struct vtn_if *vtn_if) -{ - if (vtn_if->control == SpvSelectionControlMaskNone) - return nir_selection_control_none; - else if (vtn_if->control & SpvSelectionControlDontFlattenMask) - return nir_selection_control_dont_flatten; - else if (vtn_if->control & SpvSelectionControlFlattenMask) - return nir_selection_control_flatten; - else - vtn_fail("Invalid selection control"); -} - -static void -vtn_emit_cf_list_structured(struct vtn_builder *b, struct list_head *cf_list, - nir_variable *switch_fall_var, - bool *has_switch_break, - vtn_instruction_handler handler) -{ - vtn_foreach_cf_node(node, cf_list) { - switch (node->type) { - case vtn_cf_node_type_block: { - struct vtn_block *block = vtn_cf_node_as_block(node); - - const uint32_t *block_start = block->label; - const uint32_t *block_end = block->merge ? block->merge : - block->branch; - - block_start = vtn_foreach_instruction(b, block_start, block_end, - vtn_handle_phis_first_pass); - - vtn_foreach_instruction(b, block_start, block_end, handler); - - block->end_nop = nir_nop(&b->nb); - - if (block->branch_type != vtn_branch_type_none) { - vtn_emit_branch(b, block->branch_type, block, - switch_fall_var, has_switch_break); - return; - } - - break; - } - - case vtn_cf_node_type_if: { - struct vtn_if *vtn_if = vtn_cf_node_as_if(node); - const uint32_t *branch = vtn_if->header_block->branch; - vtn_assert((branch[0] & SpvOpCodeMask) == SpvOpBranchConditional); - - bool sw_break = false; - /* If both branches are the same, just emit the first block, which is - * the only one we filled when building the CFG. - */ - if (branch[2] == branch[3]) { - if (vtn_if->then_type == vtn_branch_type_none) { - vtn_emit_cf_list_structured(b, &vtn_if->then_body, - switch_fall_var, &sw_break, handler); - } else { - vtn_emit_branch(b, vtn_if->then_type, NULL, switch_fall_var, &sw_break); - } - break; - } - - nir_if *nif = - nir_push_if(&b->nb, vtn_get_nir_ssa(b, branch[1])); - - nif->control = vtn_selection_control(b, vtn_if); - - if (vtn_if->then_type == vtn_branch_type_none) { - vtn_emit_cf_list_structured(b, &vtn_if->then_body, - switch_fall_var, &sw_break, handler); - } else { - vtn_emit_branch(b, vtn_if->then_type, NULL, switch_fall_var, &sw_break); - } - - nir_push_else(&b->nb, nif); - if (vtn_if->else_type == vtn_branch_type_none) { - vtn_emit_cf_list_structured(b, &vtn_if->else_body, - switch_fall_var, &sw_break, handler); - } else { - vtn_emit_branch(b, vtn_if->else_type, NULL, switch_fall_var, &sw_break); - } - - nir_pop_if(&b->nb, nif); - - /* If we encountered a switch break somewhere inside of the if, - * then it would have been handled correctly by calling - * emit_cf_list or emit_branch for the interrior. However, we - * need to predicate everything following on wether or not we're - * still going. - */ - if (sw_break) { - *has_switch_break = true; - nir_push_if(&b->nb, nir_load_var(&b->nb, switch_fall_var)); - } - break; - } - - case vtn_cf_node_type_loop: { - struct vtn_loop *vtn_loop = vtn_cf_node_as_loop(node); - - nir_loop *loop = nir_push_loop(&b->nb); - loop->control = vtn_loop_control(b, vtn_loop); - vtn_emit_cf_list_structured(b, &vtn_loop->body, NULL, NULL, handler); - - nir_push_continue(&b->nb, loop); - vtn_emit_cf_list_structured(b, &vtn_loop->cont_body, NULL, NULL, handler); - - nir_pop_loop(&b->nb, loop); - break; - } - - case vtn_cf_node_type_switch: { - struct vtn_switch *vtn_switch = vtn_cf_node_as_switch(node); - - /* Before we can emit anything, we need to sort the list of cases in - * fall-through order. - */ - vtn_switch_order_cases(vtn_switch); - - /* First, we create a variable to keep track of whether or not the - * switch is still going at any given point. Any switch breaks - * will set this variable to false. - */ - nir_variable *fall_var = - nir_local_variable_create(b->nb.impl, glsl_bool_type(), "fall"); - nir_store_var(&b->nb, fall_var, nir_imm_false(&b->nb), 1); - - nir_ssa_def *sel = vtn_get_nir_ssa(b, vtn_switch->selector); - - /* Now we can walk the list of cases and actually emit code */ - vtn_foreach_cf_node(case_node, &vtn_switch->cases) { - struct vtn_case *cse = vtn_cf_node_as_case(case_node); - - /* If this case jumps directly to the break block, we don't have - * to handle the case as the body is empty and doesn't fall - * through. - */ - if (cse->block == vtn_switch->break_block) - continue; - - /* Figure out the condition */ - nir_ssa_def *cond = - vtn_switch_case_condition(b, vtn_switch, sel, cse); - /* Take fallthrough into account */ - cond = nir_ior(&b->nb, cond, nir_load_var(&b->nb, fall_var)); - - nir_if *case_if = nir_push_if(&b->nb, cond); - - bool has_break = false; - nir_store_var(&b->nb, fall_var, nir_imm_true(&b->nb), 1); - vtn_emit_cf_list_structured(b, &cse->body, fall_var, &has_break, - handler); - (void)has_break; /* We don't care */ - - nir_pop_if(&b->nb, case_if); - } - - break; - } - - default: - vtn_fail("Invalid CF node type"); - } - } -} - static struct nir_block * vtn_new_unstructured_block(struct vtn_builder *b, struct vtn_function *func) { @@ -1311,7 +505,7 @@ vtn_add_unstructured_block(struct vtn_builder *b, { if (!block->block) { block->block = vtn_new_unstructured_block(b, func); - list_addtail(&block->node.link, work_list); + list_addtail(&block->link, work_list); } } @@ -1323,11 +517,11 @@ vtn_emit_cf_func_unstructured(struct vtn_builder *b, struct vtn_function *func, list_inithead(&work_list); func->start_block->block = nir_start_block(func->nir_func->impl); - list_addtail(&func->start_block->node.link, &work_list); + list_addtail(&func->start_block->link, &work_list); while (!list_is_empty(&work_list)) { struct vtn_block *block = - list_first_entry(&work_list, struct vtn_block, node.link); - list_del(&block->node.link); + list_first_entry(&work_list, struct vtn_block, link); + list_del(&block->link); vtn_assert(block->block); @@ -1369,13 +563,12 @@ vtn_emit_cf_func_unstructured(struct vtn_builder *b, struct vtn_function *func, case SpvOpSwitch: { struct list_head cases; list_inithead(&cases); - vtn_parse_switch(b, NULL, block->branch, &cases); + vtn_parse_switch(b, block->branch, &cases); nir_ssa_def *sel = vtn_get_nir_ssa(b, block->branch[1]); struct vtn_case *def = NULL; - vtn_foreach_cf_node(case_node, &cases) { - struct vtn_case *cse = vtn_cf_node_as_case(case_node); + vtn_foreach_case(cse, &cases) { if (cse->is_default) { assert(def == NULL); def = cse; @@ -1444,8 +637,7 @@ vtn_function_emit(struct vtn_builder *b, struct vtn_function *func, impl->structured = false; vtn_emit_cf_func_unstructured(b, func, instruction_handler); } else { - vtn_emit_cf_list_structured(b, &func->body, NULL, NULL, - instruction_handler); + vtn_emit_cf_func_structured(b, func, instruction_handler); } vtn_foreach_instruction(b, func->start_block->label, func->end, diff --git a/src/compiler/spirv/vtn_private.h b/src/compiler/spirv/vtn_private.h index 72d81fd0b89..465367823fe 100644 --- a/src/compiler/spirv/vtn_private.h +++ b/src/compiler/spirv/vtn_private.h @@ -41,6 +41,8 @@ extern uint32_t mesa_spirv_debug; #define MESA_SPIRV_DEBUG(flag) false #endif +#define MESA_SPIRV_DEBUG_STRUCTURED (1u << 0) + struct vtn_builder; struct vtn_decoration; @@ -134,81 +136,11 @@ enum vtn_value_type { vtn_value_type_image_pointer, }; -enum vtn_branch_type { - vtn_branch_type_none, - vtn_branch_type_if_merge, - vtn_branch_type_switch_break, - vtn_branch_type_switch_fallthrough, - vtn_branch_type_loop_break, - vtn_branch_type_loop_continue, - vtn_branch_type_loop_back_edge, - vtn_branch_type_discard, - vtn_branch_type_terminate_invocation, - vtn_branch_type_ignore_intersection, - vtn_branch_type_terminate_ray, - vtn_branch_type_emit_mesh_tasks, - vtn_branch_type_return, -}; - -enum vtn_cf_node_type { - vtn_cf_node_type_block, - vtn_cf_node_type_if, - vtn_cf_node_type_loop, - vtn_cf_node_type_case, - vtn_cf_node_type_switch, - vtn_cf_node_type_function, -}; - -struct vtn_cf_node { - struct list_head link; - struct vtn_cf_node *parent; - enum vtn_cf_node_type type; -}; - -struct vtn_loop { - struct vtn_cf_node node; - - /* The main body of the loop */ - struct list_head body; - - /* The "continue" part of the loop. This gets executed after the body - * and is where you go when you hit a continue. - */ - struct list_head cont_body; - - struct vtn_block *header_block; - struct vtn_block *cont_block; - struct vtn_block *break_block; - - SpvLoopControlMask control; -}; - -struct vtn_if { - struct vtn_cf_node node; - - enum vtn_branch_type then_type; - struct list_head then_body; - - enum vtn_branch_type else_type; - struct list_head else_body; - - struct vtn_block *header_block; - struct vtn_block *merge_block; - - SpvSelectionControlMask control; -}; - struct vtn_case { - struct vtn_cf_node node; + struct list_head link; struct vtn_block *block; - enum vtn_branch_type type; - struct list_head body; - - /* The fallthrough case, if any */ - struct vtn_case *fallthrough; - /* The uint32_t values that map to this case */ struct util_dynarray values; @@ -219,18 +151,8 @@ struct vtn_case { bool visited; }; -struct vtn_switch { - struct vtn_cf_node node; - - uint32_t selector; - - struct list_head cases; - - struct vtn_block *break_block; -}; - struct vtn_block { - struct vtn_cf_node node; + struct list_head link; /** A pointer to the label instruction */ const uint32_t *label; @@ -241,19 +163,6 @@ struct vtn_block { /** A pointer to the branch instruction that ends this block */ const uint32_t *branch; - enum vtn_branch_type branch_type; - - /* The CF node for which this is a merge target - * - * The SPIR-V spec requires that any given block can be the merge target - * for at most one merge instruction. If this block is a merge target, - * this points back to the block containing that merge instruction. - */ - struct vtn_cf_node *merge_cf_node; - - /** Points to the loop that this block starts (if it starts a loop) */ - struct vtn_loop *loop; - /** Points to the switch case started by this block (if any) */ struct vtn_case *switch_case; @@ -262,10 +171,22 @@ struct vtn_block { /** attached nir_block */ struct nir_block *block; + + /* Inner-most construct that this block is part of. */ + struct vtn_construct *parent; + + /* Blocks that succeed this block. Used by structured control flow. */ + struct vtn_successor *successors; + unsigned successors_count; + + /* Position of this block in the structured post-order traversal. */ + unsigned pos; + + bool visited; }; struct vtn_function { - struct vtn_cf_node node; + struct list_head link; struct vtn_type *type; @@ -281,25 +202,27 @@ struct vtn_function { SpvLinkageType linkage; SpvFunctionControlMask control; + + unsigned block_count; + + /* Ordering of blocks to be processed by structured control flow. See + * vtn_structured_cfg.c for details. + */ + unsigned ordered_blocks_count; + struct vtn_block **ordered_blocks; + + /* Structured control flow constructs. See struct vtn_construct. */ + struct list_head constructs; }; -#define VTN_DECL_CF_NODE_CAST(_type) \ -static inline struct vtn_##_type * \ -vtn_cf_node_as_##_type(struct vtn_cf_node *node) \ -{ \ - assert(node->type == vtn_cf_node_type_##_type); \ - return (struct vtn_##_type *)node; \ -} +#define vtn_foreach_function(func, func_list) \ + list_for_each_entry(struct vtn_function, func, func_list, link) -VTN_DECL_CF_NODE_CAST(block) -VTN_DECL_CF_NODE_CAST(loop) -VTN_DECL_CF_NODE_CAST(if) -VTN_DECL_CF_NODE_CAST(case) -VTN_DECL_CF_NODE_CAST(switch) -VTN_DECL_CF_NODE_CAST(function) +#define vtn_foreach_case(cse, case_list) \ + list_for_each_entry(struct vtn_case, cse, case_list, link) -#define vtn_foreach_cf_node(node, cf_list) \ - list_for_each_entry(struct vtn_cf_node, node, cf_list, link) +#define vtn_foreach_case_safe(cse, case_list) \ + list_for_each_entry_safe(struct vtn_case, cse, case_list, link) typedef bool (*vtn_instruction_handler)(struct vtn_builder *, SpvOp, const uint32_t *, unsigned); @@ -311,6 +234,16 @@ void vtn_function_emit(struct vtn_builder *b, struct vtn_function *func, void vtn_handle_function_call(struct vtn_builder *b, SpvOp opcode, const uint32_t *w, unsigned count); +bool vtn_cfg_handle_prepass_instruction(struct vtn_builder *b, SpvOp opcode, + const uint32_t *w, unsigned count); +void vtn_emit_cf_func_structured(struct vtn_builder *b, struct vtn_function *func, + vtn_instruction_handler handler); +bool vtn_handle_phis_first_pass(struct vtn_builder *b, SpvOp opcode, + const uint32_t *w, unsigned count); +void vtn_emit_ret_store(struct vtn_builder *b, const struct vtn_block *block); +void vtn_build_structured_cfg(struct vtn_builder *b, const uint32_t *words, + const uint32_t *end); + const uint32_t * vtn_foreach_instruction(struct vtn_builder *b, const uint32_t *start, const uint32_t *end, vtn_instruction_handler handler); @@ -906,6 +839,12 @@ vtn_get_type(struct vtn_builder *b, uint32_t value_id) return vtn_value(b, value_id, vtn_value_type_type)->type; } +static inline struct vtn_block * +vtn_block(struct vtn_builder *b, uint32_t value_id) +{ + return vtn_value(b, value_id, vtn_value_type_block)->block; +} + struct vtn_ssa_value *vtn_ssa_value(struct vtn_builder *b, uint32_t value_id); struct vtn_value *vtn_push_ssa_value(struct vtn_builder *b, uint32_t value_id, struct vtn_ssa_value *ssa); @@ -1081,4 +1020,10 @@ cmp_uint32_t(const void *pa, const void *pb) return 0; } +void +vtn_parse_switch(struct vtn_builder *b, + const uint32_t *branch, + struct list_head *case_list); + + #endif /* _VTN_PRIVATE_H_ */ diff --git a/src/compiler/spirv/vtn_structured_cfg.c b/src/compiler/spirv/vtn_structured_cfg.c new file mode 100644 index 00000000000..07ad125e15f --- /dev/null +++ b/src/compiler/spirv/vtn_structured_cfg.c @@ -0,0 +1,1735 @@ +/* + * Copyright © 2015-2023 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "vtn_private.h" +#include "spirv_info.h" +#include "util/u_math.h" + +/* Handle SPIR-V structured control flow, mapping SPIR-V constructs into + * equivalent NIR constructs. + * + * Because SPIR-V can represent more complex control flow than NIR, some + * constructs are mapped into a combination of nir_if and nir_loop nodes. For + * example, an selection construct with an "if-break" (an early branch into + * the end of the construct) will be mapped into NIR as a loop (to allow the + * break) with a nested if (to handle the actual selection). + * + * Note that using NIR loops this way requires us to propagate breaks and + * continues that are meant to outer constructs when a nir_loop is used for a + * SPIR-V construct other than Loop. + * + * The process of identifying and ordering the blocks before the NIR + * translation is similar to what's done in Tint, using the "reverse + * structured post-order traversal". See also the file comments + * src/reader/spirv/function.cc in the Tint repository. + */ + +enum vtn_construct_type { + /* Not formally a SPIR-V construct but used to represent the entire + * function. + */ + vtn_construct_type_function, + + /* Selection construct uses a nir_if and optionally a nir_loop to handle + * if-breaks. + */ + vtn_construct_type_selection, + + /* Loop construct uses a nir_loop and optionally a nir_if to handle an + * OpBranchConditional as part of the head of the loop. + */ + vtn_construct_type_loop, + + /* Continue construct maps to the NIR continue construct of the corresponding + * loop. For convenience, unlike in SPIR-V, the parent of this construct is + * always the loop construct. Continue construct is omitted for single-block + * loops. + */ + vtn_construct_type_continue, + + /* Switch construct is not directly mapped into any NIR structure, the work + * is handled by the case constructs. It does keep a nir_variable for + * handling case fallback logic. + */ + vtn_construct_type_switch, + + /* Case construct uses a nir_if and optionally a nir_loop to handle early + * breaks. Note switch_breaks are handled by each case. + */ + vtn_construct_type_case, +}; + +static const char * +vtn_construct_type_to_string(enum vtn_construct_type t) +{ +#define CASE(typ) case vtn_construct_type_##typ: return #typ + switch (t) { + CASE(function); + CASE(selection); + CASE(loop); + CASE(continue); + CASE(switch); + CASE(case); + } +#undef CASE + unreachable("invalid construct type"); + return ""; +} + +struct vtn_construct { + enum vtn_construct_type type; + + bool needs_nloop; + bool needs_break_propagation; + bool needs_continue_propagation; + bool needs_fallthrough; + + struct vtn_construct *parent; + + struct vtn_construct *innermost_loop; + struct vtn_construct *innermost_switch; + struct vtn_construct *innermost_case; + + unsigned start_pos; + unsigned end_pos; + + /* Usually the same as end_pos, but may be different in case of an "early + * merge" after divergence caused by an OpBranchConditional. This can + * happen in selection and loop constructs. + */ + unsigned merge_pos; + + /* Valid when not zero, indicates the block that starts the then and else + * paths in a condition. This may be used by selection constructs. + */ + unsigned then_pos; + unsigned else_pos; + + /* Indicates where the continue block is, marking the end of the body of + * the loop. Note the block ordering will always give us first the loop + * body blocks then the continue block. Used by loop construct. + */ + unsigned continue_pos; + + /* For the list of all constructs in vtn_function. */ + struct list_head link; + + /* NIR nodes that are associated with this construct. See + * vtn_construct_type for an overview. + */ + nir_loop *nloop; + nir_if *nif; + + /* This variable will be set by an inner construct to indicate that a break + * is necessary. We need to use variables here for situations when the + * inner construct has a loop of its own for other reasons. + */ + nir_variable *break_var; + + /* Same logic but for continue. */ + nir_variable *continue_var; + + /* This is used by each case to force entering in the case regardless of + * the condition. We always set it when handling a branch that is a + * switch_break or a switch_fallthrough. + */ + nir_variable *fallthrough_var; + + unsigned index; +}; + +enum vtn_branch_type { + vtn_branch_type_none, + vtn_branch_type_forward, + vtn_branch_type_if_break, + vtn_branch_type_switch_break, + vtn_branch_type_switch_fallthrough, + vtn_branch_type_loop_break, + vtn_branch_type_loop_continue, + vtn_branch_type_loop_back_edge, + vtn_branch_type_discard, + vtn_branch_type_terminate_invocation, + vtn_branch_type_ignore_intersection, + vtn_branch_type_terminate_ray, + vtn_branch_type_emit_mesh_tasks, + vtn_branch_type_return, +}; + +static const char * +vtn_branch_type_to_string(enum vtn_branch_type t) +{ +#define CASE(typ) case vtn_branch_type_##typ: return #typ + switch (t) { + CASE(none); + CASE(forward); + CASE(if_break); + CASE(switch_break); + CASE(switch_fallthrough); + CASE(loop_break); + CASE(loop_continue); + CASE(loop_back_edge); + CASE(discard); + CASE(terminate_invocation); + CASE(ignore_intersection); + CASE(terminate_ray); + CASE(emit_mesh_tasks); + CASE(return); + } +#undef CASE + unreachable("unknown branch type"); + return ""; +} + +struct vtn_successor { + struct vtn_block *block; + enum vtn_branch_type branch_type; +}; + +static bool +vtn_is_single_block_loop(const struct vtn_construct *c) +{ + return c->type == vtn_construct_type_loop && + c->start_pos == c->continue_pos; +} + +static struct vtn_construct * +vtn_find_innermost(enum vtn_construct_type type, struct vtn_construct *c) +{ + while (c && c->type != type) + c = c->parent; + return c; +} + +static void +print_ordered_blocks(const struct vtn_function *func) +{ + for (unsigned i = 0; i < func->ordered_blocks_count; i++) { + struct vtn_block *block = func->ordered_blocks[i]; + printf("[id=%-6u] %4u", block->label[1], block->pos); + if (block->successors_count > 0) { + printf(" ->"); + for (unsigned j = 0; j < block->successors_count; j++) { + printf(" "); + if (block->successors[j].block) + printf("%u/", block->successors[j].block->pos); + printf("%s", vtn_branch_type_to_string(block->successors[j].branch_type)); + } + } + if (!block->visited) + printf(" NOT VISITED"); + printf("\n"); + } +} + +static struct vtn_case * +vtn_find_fallthrough_target(struct vtn_builder *b, const uint32_t *switch_merge, + struct vtn_block *source_block, struct vtn_block *block) +{ + if (block->visited) + return NULL; + + if (block->label[1] == switch_merge[1]) + return NULL; + + /* Don't consider the initial source block a fallthrough target of itself. */ + if (block->switch_case && block != source_block) + return block->switch_case; + + if (block->merge) + return vtn_find_fallthrough_target(b, switch_merge, source_block, + vtn_block(b, block->merge[1])); + + const uint32_t *branch = block->branch; + vtn_assert(branch); + + switch (branch[0] & SpvOpCodeMask) { + case SpvOpBranch: + return vtn_find_fallthrough_target(b, switch_merge, source_block, + vtn_block(b, branch[1])); + case SpvOpBranchConditional: { + struct vtn_case *target = + vtn_find_fallthrough_target(b, switch_merge, source_block, + vtn_block(b, branch[2])); + if (!target) + target = vtn_find_fallthrough_target(b, switch_merge, source_block, + vtn_block(b, branch[3])); + return target; + } + default: + return NULL; + } +} + +static void +structured_post_order_traversal(struct vtn_builder *b, struct vtn_block *block) +{ + if (block->visited) + return; + + block->visited = true; + + if (block->merge) { + structured_post_order_traversal(b, vtn_block(b, block->merge[1])); + + SpvOp merge_op = block->merge[0] & SpvOpCodeMask; + if (merge_op == SpvOpLoopMerge) { + struct vtn_block *continue_block = vtn_block(b, block->merge[2]); + structured_post_order_traversal(b, continue_block); + } + } + + const uint32_t *branch = block->branch; + vtn_assert(branch); + + switch (branch[0] & SpvOpCodeMask) { + case SpvOpBranch: + block->successors_count = 1; + block->successors = rzalloc(b, struct vtn_successor); + block->successors[0].block = vtn_block(b, branch[1]); + structured_post_order_traversal(b, block->successors[0].block); + break; + + case SpvOpBranchConditional: + block->successors_count = 2; + block->successors = rzalloc_array(b, struct vtn_successor, 2); + block->successors[0].block = vtn_block(b, branch[2]); + block->successors[1].block = vtn_block(b, branch[3]); + + /* The result of the traversal will be reversed, so to provide a + * more natural order, with THEN blocks appearing before ELSE blocks, + * we need to traverse them in the reversed order. + */ + int order[] = { 1, 0 }; + + /* There's a catch when traversing case fallthroughs: we want to avoid + * walking part of a case construct, then the fallthrough -- possibly + * visiting another entire case construct, and back to the other part + * of that original case construct. So if the THEN path is a fallthrough, + * swap the visit order. + */ + if (block->successors[0].block->switch_case) { + order[0] = !order[0]; + order[1] = !order[1]; + } + + structured_post_order_traversal(b, block->successors[order[0]].block); + structured_post_order_traversal(b, block->successors[order[1]].block); + break; + + case SpvOpSwitch: { + /* TODO: Save this to use during Switch construct creation. */ + struct list_head cases; + list_inithead(&cases); + vtn_parse_switch(b, block->branch, &cases); + + block->successors_count = list_length(&cases); + block->successors = rzalloc_array(b, struct vtn_successor, block->successors_count); + + /* The 'Rules for Structured Control-flow constructs' already guarantee + * that the labels of the targets are ordered in a way that if + * there is a fallthrough, they will appear consecutively. The only + * exception is Default, which is always the first in the list. + * + * Because we are doing a DFS from the end of the cases, the + * traversal already handle a Case falling through Default. + * + * The scenario that needs fixing is when no case falls to Default, but + * Default falls to another case. For that scenario we move the Default + * right before the case it falls to. + */ + + struct vtn_case *default_case = list_first_entry(&cases, struct vtn_case, link); + vtn_assert(default_case && default_case->is_default); + + struct vtn_case *fall_target = + vtn_find_fallthrough_target(b, block->merge, default_case->block, + default_case->block); + if (fall_target) + list_move_to(&default_case->link, &fall_target->link); + + /* Because the result of the traversal will be reversed, loop backwards + * in the case list. + */ + unsigned i = 0; + list_for_each_entry_rev(struct vtn_case, cse, &cases, link) { + structured_post_order_traversal(b, cse->block); + block->successors[i].block = cse->block; + i++; + } + + break; + } + + case SpvOpKill: + case SpvOpTerminateInvocation: + case SpvOpIgnoreIntersectionKHR: + case SpvOpTerminateRayKHR: + case SpvOpReturn: + case SpvOpReturnValue: + case SpvOpEmitMeshTasksEXT: + case SpvOpUnreachable: + block->successors_count = 1; + block->successors = rzalloc(b, struct vtn_successor); + break; + + default: + unreachable("invalid branch opcode"); + } + + b->func->ordered_blocks[b->func->ordered_blocks_count++] = block; +} + +static void +sort_blocks(struct vtn_builder *b) +{ + struct vtn_block **ordered_blocks = + rzalloc_array(b, struct vtn_block *, b->func->block_count); + + b->func->ordered_blocks = ordered_blocks; + + structured_post_order_traversal(b, b->func->start_block); + + /* Reverse it, so that blocks appear before their successors. */ + unsigned count = b->func->ordered_blocks_count; + for (unsigned i = 0; i < (count / 2); i++) { + unsigned j = count - i - 1; + struct vtn_block *tmp = ordered_blocks[i]; + ordered_blocks[i] = ordered_blocks[j]; + ordered_blocks[j] = tmp; + } + + for (unsigned i = 0; i < count; i++) + ordered_blocks[i]->pos = i; +} + +static void +print_construct(const struct vtn_function *func, + const struct vtn_construct *c) +{ + for (const struct vtn_construct *p = c->parent; p; p = p->parent) + printf(" "); + printf("C%u/%s ", c->index, vtn_construct_type_to_string(c->type)); + printf(" %u->%u", c->start_pos, c->end_pos); + if (c->merge_pos) + printf(" merge=%u", c->merge_pos); + if (c->then_pos) + printf(" then=%u", c->then_pos); + if (c->else_pos) + printf(" else=%u", c->else_pos); + if (c->needs_nloop) + printf(" nloop"); + if (c->needs_break_propagation) + printf(" break_prop"); + if (c->needs_continue_propagation) + printf(" continue_prop"); + if (c->type == vtn_construct_type_loop) { + if (vtn_is_single_block_loop(c)) + printf(" single_block_loop"); + else + printf(" cont=%u", c->continue_pos); + } + if (c->type == vtn_construct_type_case) { + struct vtn_block *block = func->ordered_blocks[c->start_pos]; + if (block->switch_case->is_default) { + printf(" [default]"); + } else { + printf(" [values:"); + util_dynarray_foreach(&block->switch_case->values, uint64_t, val) + printf(" %" PRIu64, *val); + printf("]"); + } + } + printf("\n"); +} + +static void +print_constructs(struct vtn_function *func) +{ + list_for_each_entry(struct vtn_construct, c, &func->constructs, link) + print_construct(func, c); +} + +struct vtn_construct_stack { + /* Array of `struct vtn_construct *`. */ + struct util_dynarray data; +}; + +static inline void +init_construct_stack(struct vtn_construct_stack *stack, void *mem_ctx) +{ + assert(mem_ctx); + util_dynarray_init(&stack->data, mem_ctx); +} + +static inline unsigned +count_construct_stack(struct vtn_construct_stack *stack) +{ + return util_dynarray_num_elements(&stack->data, struct vtn_construct *); +} + +static inline struct vtn_construct * +top_construct(struct vtn_construct_stack *stack) +{ + assert(count_construct_stack(stack) > 0); + return util_dynarray_top(&stack->data, struct vtn_construct *); +} + +static inline void +pop_construct(struct vtn_construct_stack *stack) +{ + assert(count_construct_stack(stack) > 0); + (void)util_dynarray_pop(&stack->data, struct vtn_construct *); +} + +static inline void +push_construct(struct vtn_construct_stack *stack, struct vtn_construct *c) +{ + util_dynarray_append(&stack->data, struct vtn_construct *, c); +} + +static int +cmp_succ_block_pos(const void *pa, const void *pb) +{ + const struct vtn_successor *sa = pa; + const struct vtn_successor *sb = pb; + const unsigned a = sa->block->pos; + const unsigned b = sb->block->pos; + if (a < b) + return -1; + if (a > b) + return 1; + return 0; +} + +static void +create_constructs(struct vtn_builder *b) +{ + struct vtn_construct *func_construct = rzalloc(b, struct vtn_construct); + func_construct->type = vtn_construct_type_function; + func_construct->start_pos = 0; + func_construct->end_pos = b->func->ordered_blocks_count; + + for (unsigned i = 0; i < b->func->ordered_blocks_count; i++) { + struct vtn_block *block = b->func->ordered_blocks[i]; + + if (block->merge) { + SpvOp merge_op = block->merge[0] & SpvOpCodeMask; + SpvOp branch_op = block->branch[0] & SpvOpCodeMask; + + const unsigned end_pos = vtn_block(b, block->merge[1])->pos; + + if (merge_op == SpvOpLoopMerge) { + struct vtn_construct *loop = rzalloc(b, struct vtn_construct); + loop->type = vtn_construct_type_loop; + loop->start_pos = block->pos; + loop->end_pos = end_pos; + + loop->parent = block->parent; + block->parent = loop; + + struct vtn_block *continue_block = vtn_block(b, block->merge[2]); + loop->continue_pos = continue_block->pos; + + if (!vtn_is_single_block_loop(loop)) { + struct vtn_construct *cont = rzalloc(b, struct vtn_construct); + cont->type = vtn_construct_type_continue; + cont->parent = loop; + cont->start_pos = loop->continue_pos; + cont->end_pos = end_pos; + + cont->parent = loop; + continue_block->parent = cont; + } + + /* Not all combinations of OpLoopMerge and OpBranchConditional are valid, + * workaround for invalid combinations by injecting an extra selection. + * + * Old versions of dxil-spirv generated this. + */ + if (branch_op == SpvOpBranchConditional) { + vtn_assert(block->successors_count == 2); + const unsigned then_pos = block->successors[0].block ? + block->successors[0].block->pos : 0; + const unsigned else_pos = block->successors[1].block ? + block->successors[1].block->pos : 0; + + if (then_pos > loop->start_pos && then_pos < loop->continue_pos && + else_pos > loop->start_pos && else_pos < loop->continue_pos) { + vtn_warn("An OpSelectionMerge instruction is required to precede " + "an OpBranchConditional instruction that has different " + "True Label and False Label operands where neither are " + "declared merge blocks or Continue Targets."); + struct vtn_construct *sel = rzalloc(b, struct vtn_construct); + sel->type = vtn_construct_type_selection; + sel->start_pos = loop->start_pos; + sel->end_pos = loop->continue_pos; + sel->then_pos = then_pos; + sel->else_pos = else_pos; + sel->parent = loop; + block->parent = sel; + } + } + + } else if (branch_op == SpvOpSwitch) { + vtn_assert(merge_op == SpvOpSelectionMerge); + + struct vtn_construct *swtch = rzalloc(b, struct vtn_construct); + swtch->type = vtn_construct_type_switch; + swtch->start_pos = block->pos; + swtch->end_pos = end_pos; + + swtch->parent = block->parent; + block->parent = swtch; + + struct list_head cases; + list_inithead(&cases); + vtn_parse_switch(b, block->branch, &cases); + + vtn_foreach_case_safe(cse, &cases) { + if (cse->block->pos < end_pos) { + struct vtn_block *case_block = cse->block; + struct vtn_construct *c = rzalloc(b, struct vtn_construct); + c->type = vtn_construct_type_case; + c->parent = swtch; + c->start_pos = case_block->pos; + + /* Upper bound, will be updated right after. */ + c->end_pos = swtch->end_pos; + + vtn_assert(case_block->parent == NULL || case_block->parent == swtch); + case_block->parent = c; + } else { + /* A target in OpSwitch must point either to one of the case + * constructs or to the Merge block. No outer break/continue + * is allowed. + */ + vtn_assert(cse->block->pos == end_pos); + } + list_delinit(&cse->link); + } + + /* Case constructs don't overlap, so they end as the next one + * begins. + */ + qsort(block->successors, block->successors_count, + sizeof(struct vtn_successor), cmp_succ_block_pos); + for (unsigned succ_idx = 1; succ_idx < block->successors_count; succ_idx++) { + unsigned succ_pos = block->successors[succ_idx].block->pos; + /* The successors are ordered, so once we see a successor point + * to the merge block, we are done fixing the cases. + */ + if (succ_pos >= swtch->end_pos) + break; + struct vtn_construct *prev_cse = + vtn_find_innermost(vtn_construct_type_case, + block->successors[succ_idx - 1].block->parent); + vtn_assert(prev_cse); + prev_cse->end_pos = succ_pos; + } + + } else { + vtn_assert(merge_op == SpvOpSelectionMerge); + vtn_assert(branch_op == SpvOpBranchConditional); + + struct vtn_construct *sel = rzalloc(b, struct vtn_construct); + sel->type = vtn_construct_type_selection; + sel->start_pos = block->pos; + sel->end_pos = end_pos; + sel->parent = block->parent; + block->parent = sel; + + vtn_assert(block->successors_count == 2); + struct vtn_block *then_block = block->successors[0].block; + struct vtn_block *else_block = block->successors[1].block; + + sel->then_pos = then_block ? then_block->pos : 0; + sel->else_pos = else_block ? else_block->pos : 0; + } + } + } + + /* Link the constructs with their parents and with the remaining blocks + * that do not start one. This will also build the ordered list of + * constructs. + */ + struct vtn_construct_stack stack; + init_construct_stack(&stack, b); + push_construct(&stack, func_construct); + list_addtail(&func_construct->link, &b->func->constructs); + + for (unsigned i = 0; i < b->func->ordered_blocks_count; i++) { + struct vtn_block *block = b->func->ordered_blocks[i]; + + while (block->pos == top_construct(&stack)->end_pos) + pop_construct(&stack); + + /* Identify the start of a continue construct. */ + if (top_construct(&stack)->type == vtn_construct_type_loop && + !vtn_is_single_block_loop(top_construct(&stack)) && + top_construct(&stack)->continue_pos == block->pos) { + struct vtn_construct *c = vtn_find_innermost(vtn_construct_type_continue, block->parent); + vtn_assert(c); + vtn_assert(c->parent == top_construct(&stack)); + + list_addtail(&c->link, &b->func->constructs); + push_construct(&stack, c); + } + + if (top_construct(&stack)->type == vtn_construct_type_switch) { + struct vtn_block *header = b->func->ordered_blocks[top_construct(&stack)->start_pos]; + for (unsigned succ_idx = 0; succ_idx < header->successors_count; succ_idx++) { + struct vtn_successor *succ = &header->successors[succ_idx]; + if (block == succ->block) { + struct vtn_construct *c = vtn_find_innermost(vtn_construct_type_case, succ->block->parent); + if (c) { + vtn_assert(c->parent == top_construct(&stack)); + + list_addtail(&c->link, &b->func->constructs); + push_construct(&stack, c); + } + break; + } + } + } + + if (block->merge) { + switch (block->merge[0] & SpvOpCodeMask) { + case SpvOpSelectionMerge: { + struct vtn_construct *c = block->parent; + vtn_assert(c->type == vtn_construct_type_selection || + c->type == vtn_construct_type_switch); + + c->parent = top_construct(&stack); + + list_addtail(&c->link, &b->func->constructs); + push_construct(&stack, c); + break; + } + + case SpvOpLoopMerge: { + struct vtn_construct *c = block->parent; + struct vtn_construct *loop = c; + + /* A loop might have an extra selection injected, skip it. */ + if (c->type == vtn_construct_type_selection) + loop = c->parent; + + vtn_assert(loop->type == vtn_construct_type_loop); + loop->parent = top_construct(&stack); + + list_addtail(&loop->link, &b->func->constructs); + push_construct(&stack, loop); + + if (loop != c) { + /* Make sure we also "enter" the extra construct. */ + list_addtail(&c->link, &b->func->constructs); + push_construct(&stack, c); + } + break; + } + + default: + unreachable("invalid merge opcode"); + } + } + + block->parent = top_construct(&stack); + } + + vtn_assert(count_construct_stack(&stack) == 1); + vtn_assert(top_construct(&stack)->type == vtn_construct_type_function); + + unsigned index = 0; + list_for_each_entry(struct vtn_construct, c, &b->func->constructs, link) + c->index = index++; +} + +static void +validate_constructs(struct vtn_builder *b) +{ + list_for_each_entry(struct vtn_construct, c, &b->func->constructs, link) { + if (c->type == vtn_construct_type_function) + vtn_assert(c->parent == NULL); + else + vtn_assert(c->parent); + + switch (c->type) { + case vtn_construct_type_continue: + vtn_assert(c->parent->type == vtn_construct_type_loop); + break; + case vtn_construct_type_case: + vtn_assert(c->parent->type == vtn_construct_type_switch); + break; + default: + /* Nothing to do. */ + break; + } + } +} + +static void +find_innermost_constructs(struct vtn_builder *b) +{ + list_for_each_entry(struct vtn_construct, c, &b->func->constructs, link) { + if (c->type == vtn_construct_type_function) { + c->innermost_loop = NULL; + c->innermost_switch = NULL; + c->innermost_case = NULL; + continue; + } + + if (c->type == vtn_construct_type_loop) + c->innermost_loop = c; + else + c->innermost_loop = c->parent->innermost_loop; + + if (c->type == vtn_construct_type_switch) + c->innermost_switch = c; + else + c->innermost_switch = c->parent->innermost_switch; + + if (c->type == vtn_construct_type_case) + c->innermost_case = c; + else + c->innermost_case = c->parent->innermost_case; + } + + list_for_each_entry(struct vtn_construct, c, &b->func->constructs, link) { + vtn_assert(vtn_find_innermost(vtn_construct_type_loop, c) == c->innermost_loop); + vtn_assert(vtn_find_innermost(vtn_construct_type_switch, c) == c->innermost_switch); + vtn_assert(vtn_find_innermost(vtn_construct_type_case, c) == c->innermost_case); + } +} + +static void +set_needs_continue_propagation(struct vtn_construct *c) +{ + for (; c != c->innermost_loop; c = c->parent) + c->needs_continue_propagation = true; +} + +static void +set_needs_break_propagation(struct vtn_construct *c, + struct vtn_construct *to_break) +{ + for (; c != to_break; c = c->parent) + c->needs_break_propagation = true; +} + +static enum vtn_branch_type +branch_type_for_successor(struct vtn_builder *b, struct vtn_block *block, + struct vtn_successor *succ) +{ + unsigned pos = block->pos; + unsigned succ_pos = succ->block->pos; + + struct vtn_construct *inner = block->parent; + vtn_assert(inner); + + /* Identify the types of branches, applying the "Rules for Structured + * Control-flow Constructs" from SPIR-V spec. + */ + + struct vtn_construct *innermost_loop = inner->innermost_loop; + if (innermost_loop) { + /* Entering the innermost loop’s continue construct. */ + if (!vtn_is_single_block_loop(innermost_loop) && + succ_pos == innermost_loop->continue_pos) { + set_needs_continue_propagation(inner); + return vtn_branch_type_loop_continue; + } + + /* Breaking from the innermost loop (and branching from back-edge block + * to loop merge). + */ + if (succ_pos == innermost_loop->end_pos) { + set_needs_break_propagation(inner, innermost_loop); + return vtn_branch_type_loop_break; + } + + /* Next loop iteration. There can be only a single loop back-edge + * for each loop construct. + */ + if (succ_pos == innermost_loop->start_pos) { + vtn_assert(inner->type == vtn_construct_type_continue || + vtn_is_single_block_loop(innermost_loop)); + return vtn_branch_type_loop_back_edge; + } + } + + struct vtn_construct *innermost_switch = inner->innermost_switch; + if (innermost_switch) { + struct vtn_construct *innermost_cse = inner->innermost_case; + + /* Breaking from the innermost switch construct. */ + if (succ_pos == innermost_switch->end_pos) { + /* Use a nloop if this is not a natural exit from a case construct. */ + if (innermost_cse && pos != innermost_cse->end_pos - 1) { + innermost_cse->needs_nloop = true; + set_needs_break_propagation(inner, innermost_cse); + } + return vtn_branch_type_switch_break; + } + + /* Branching from one case construct to another. */ + if (inner != innermost_switch) { + vtn_assert(innermost_cse); + vtn_assert(innermost_cse->parent == innermost_switch); + + if (succ->block->switch_case) { + /* Both cases should be from the same Switch construct. */ + struct vtn_construct *target_cse = succ->block->parent->innermost_case; + vtn_assert(target_cse->parent == innermost_switch); + target_cse->needs_fallthrough = true; + return vtn_branch_type_switch_fallthrough; + } + } + } + + if (inner->type == vtn_construct_type_selection) { + /* Branches from the header block that were not categorized above will + * follow to the then/else paths or to the merge block, and are handled + * by the nir_if node. + */ + if (block->merge) + return vtn_branch_type_forward; + + /* Breaking from a selection construct. */ + if (succ_pos == inner->end_pos) { + /* Identify cases where the break would be a natural flow in the NIR + * construct. We don't need the extra loop in such cases. + * + * Because then/else are not ordered, we need to find which one happens + * later. For non early merges, the branch from the block right before + * the second side of the if starts will also jumps naturally to the + * end of the if. + */ + const bool has_early_merge = inner->merge_pos != inner->end_pos; + const unsigned second_pos = MAX2(inner->then_pos, inner->else_pos); + + const bool natural_exit_from_if = + pos + 1 == inner->end_pos || + (!has_early_merge && (pos + 1 == second_pos)); + + inner->needs_nloop = !natural_exit_from_if; + return vtn_branch_type_if_break; + } + } + + if (succ_pos < inner->end_pos) + return vtn_branch_type_forward; + + const enum nir_spirv_debug_level level = NIR_SPIRV_DEBUG_LEVEL_ERROR; + const size_t offset = 0; + + vtn_logf(b, level, offset, + "SPIR-V parsing FAILED:\n" + " Unrecognized branch from block pos %u (id=%u) " + "to block pos %u (id=%u)", + block->pos, block->label[1], + succ->block->pos, succ->block->label[1]); + + vtn_logf(b, level, offset, + " Inner construct '%s': %u -> %u (merge=%u then=%u else=%u)", + vtn_construct_type_to_string(inner->type), + inner->start_pos, inner->end_pos, inner->merge_pos, inner->then_pos, inner->else_pos); + + struct vtn_construct *outer = inner->parent; + if (outer) { + vtn_logf(b, level, offset, + " Outer construct '%s': %u -> %u (merge=%u then=%u else=%u)", + vtn_construct_type_to_string(outer->type), + outer->start_pos, outer->end_pos, outer->merge_pos, outer->then_pos, outer->else_pos); + } + + vtn_fail("Unable to identify branch type"); + return vtn_branch_type_none; +} + +static enum vtn_branch_type +branch_type_for_terminator(struct vtn_builder *b, struct vtn_block *block) +{ + vtn_assert(block->successors_count == 1); + vtn_assert(block->successors[0].block == NULL); + + switch (block->branch[0] & SpvOpCodeMask) { + case SpvOpKill: + return vtn_branch_type_discard; + case SpvOpTerminateInvocation: + return vtn_branch_type_terminate_invocation; + case SpvOpIgnoreIntersectionKHR: + return vtn_branch_type_ignore_intersection; + case SpvOpTerminateRayKHR: + return vtn_branch_type_terminate_ray; + case SpvOpEmitMeshTasksEXT: + return vtn_branch_type_emit_mesh_tasks; + case SpvOpReturn: + case SpvOpReturnValue: + case SpvOpUnreachable: + return vtn_branch_type_return; + default: + unreachable("unexpected terminator operation"); + return vtn_branch_type_none; + } +} + +static void +set_branch_types(struct vtn_builder *b) +{ + for (unsigned i = 0; i < b->func->ordered_blocks_count; i++) { + struct vtn_block *block = b->func->ordered_blocks[i]; + for (unsigned j = 0; j < block->successors_count; j++) { + struct vtn_successor *succ = &block->successors[j]; + + if (succ->block) + succ->branch_type = branch_type_for_successor(b, block, succ); + else + succ->branch_type = branch_type_for_terminator(b, block); + + vtn_assert(succ->branch_type != vtn_branch_type_none); + } + } +} + +static void +find_merge_pos(struct vtn_builder *b) +{ + /* Merges are at the end of the construct by construction... */ + list_for_each_entry(struct vtn_construct, c, &b->func->constructs, link) + c->merge_pos = c->end_pos; + + /* ...except when we have an "early merge", i.e. a branch that converges + * before the declared merge point. For these cases the actual merge is + * stored in merge_pos. + * + * Look at all header blocks for constructs that may have such early + * merge, and check whether they fit + */ + for (unsigned i = 0; i < b->func->ordered_blocks_count; i++) { + if (!b->func->ordered_blocks[i]->merge) + continue; + + struct vtn_block *header = b->func->ordered_blocks[i]; + if (header->successors_count != 2) + continue; + + /* Ignore single-block loops (i.e. header thats in a continue + * construct). Because the loop has no body, no block will + * be identified in the then/else sides, the vtn_emit_branch + * calls will be enough. + */ + + struct vtn_construct *c = header->parent; + if (c->type != vtn_construct_type_selection) + continue; + + const unsigned first_pos = MIN2(c->then_pos, c->else_pos); + const unsigned second_pos = MAX2(c->then_pos, c->else_pos); + + /* The first side ends where the second starts. The second side ends + * either the continue position (that is guaranteed to appear after the + * body of a loop) or the actual end of the construct. + * + * Because of the way we ordered the blocks, if there's an early merge, + * the first side of the if will have a branch inside the second side. + */ + const unsigned first_end = second_pos; + const unsigned second_end = c->end_pos; + + unsigned early_merge_pos = 0; + for (unsigned pos = first_pos; pos < first_end; pos++) { + /* For each block in first... */ + struct vtn_block *block = b->func->ordered_blocks[pos]; + for (unsigned s = 0; s < block->successors_count; s++) { + if (block->successors[s].block) { + /* ...see if one of its successors branches to the second side. */ + const unsigned succ_pos = block->successors[s].block->pos; + if (succ_pos >= second_pos && succ_pos < second_end) { + vtn_fail_if(early_merge_pos, + "A single selection construct cannot " + "have multiple early merges"); + early_merge_pos = succ_pos; + } + } + } + + if (early_merge_pos) { + c->merge_pos = early_merge_pos; + break; + } + } + } +} + +void +vtn_build_structured_cfg(struct vtn_builder *b, const uint32_t *words, const uint32_t *end) +{ + vtn_foreach_function(func, &b->functions) { + b->func = func; + + sort_blocks(b); + + create_constructs(b); + + validate_constructs(b); + + find_innermost_constructs(b); + + find_merge_pos(b); + + set_branch_types(b); + + if (MESA_SPIRV_DEBUG(STRUCTURED)) { + printf("\nBLOCKS (%u):\n", func->ordered_blocks_count); + print_ordered_blocks(func); + printf("\nCONSTRUCTS (%u):\n", list_length(&func->constructs)); + print_constructs(func); + printf("\n"); + } + } +} + +static int +vtn_set_break_vars_between(struct vtn_builder *b, + struct vtn_construct *from, + struct vtn_construct *to) +{ + vtn_assert(from); + vtn_assert(to); + + int count = 0; + for (struct vtn_construct *c = from; c != to; c = c->parent) { + if (c->break_var) { + vtn_assert(c->nloop); + count++; + + /* There's no need to set break_var for the from block an actual break will be emitted + * by the callers. + */ + if (c != from) + nir_store_var(&b->nb, c->break_var, nir_imm_true(&b->nb), 1); + } else { + /* There's a 1:1 correspondence between break_vars and nloops. */ + vtn_assert(!c->nloop); + } + } + + return count; +} + +static void +vtn_emit_break_for_construct(struct vtn_builder *b, + const struct vtn_block *block, + struct vtn_construct *to_break) +{ + vtn_assert(to_break); + vtn_assert(to_break->nloop); + + bool has_intermediate = vtn_set_break_vars_between(b, block->parent, to_break); + if (has_intermediate) + nir_store_var(&b->nb, to_break->break_var, nir_imm_true(&b->nb), 1); + + nir_jump(&b->nb, nir_jump_break); +} + +static void +vtn_emit_continue_for_construct(struct vtn_builder *b, + const struct vtn_block *block, + struct vtn_construct *to_continue) +{ + vtn_assert(to_continue); + vtn_assert(to_continue->type == vtn_construct_type_loop); + vtn_assert(to_continue->nloop); + + bool has_intermediate = vtn_set_break_vars_between(b, block->parent, to_continue); + if (has_intermediate) { + nir_store_var(&b->nb, to_continue->continue_var, nir_imm_true(&b->nb), 1); + nir_jump(&b->nb, nir_jump_break); + } else { + nir_jump(&b->nb, nir_jump_continue); + } +} + +static void +vtn_emit_branch(struct vtn_builder *b, const struct vtn_block *block, + const struct vtn_successor *succ) +{ + switch (succ->branch_type) { + case vtn_branch_type_none: + vtn_assert(!"invalid branch type"); + break; + + case vtn_branch_type_forward: + /* Nothing to do. */ + break; + + case vtn_branch_type_if_break: { + struct vtn_construct *inner_if = block->parent; + vtn_assert(inner_if->type == vtn_construct_type_selection); + if (inner_if->nloop) { + vtn_emit_break_for_construct(b, block, inner_if); + } else { + /* Nothing to do. This is a natural exit from an if construct. */ + } + break; + } + + case vtn_branch_type_switch_break: { + struct vtn_construct *swtch = block->parent->innermost_switch; + vtn_assert(swtch); + + struct vtn_construct *cse = block->parent->innermost_case; + if (cse && cse->parent == swtch && cse->nloop) { + vtn_emit_break_for_construct(b, block, cse); + } else { + /* Nothing to do. This case doesn't have a loop, so this is a + * natural break from a case. + */ + } + break; + } + + case vtn_branch_type_switch_fallthrough: { + struct vtn_construct *cse = block->parent->innermost_case; + vtn_assert(cse); + + struct vtn_construct *swtch = cse->parent; + vtn_assert(swtch->type == vtn_construct_type_switch); + + /* Successor is the start of another case construct with the same parent + * switch construct. + */ + vtn_assert(succ->block->switch_case != NULL); + struct vtn_construct *target = succ->block->parent->innermost_case; + vtn_assert(target != NULL && target->type == vtn_construct_type_case); + vtn_assert(target->parent == swtch); + vtn_assert(target->fallthrough_var); + + nir_store_var(&b->nb, target->fallthrough_var, nir_imm_true(&b->nb), 1); + if (cse->nloop) + vtn_emit_break_for_construct(b, block, cse); + break; + } + + case vtn_branch_type_loop_break: { + struct vtn_construct *loop = block->parent->innermost_loop; + vtn_assert(loop); + vtn_emit_break_for_construct(b, block, loop); + break; + } + + case vtn_branch_type_loop_continue: { + struct vtn_construct *loop = block->parent->innermost_loop; + vtn_assert(loop); + vtn_emit_continue_for_construct(b, block, loop); + break; + } + + case vtn_branch_type_loop_back_edge: + /* Nothing to do: naturally handled by NIR loop node. */ + break; + + case vtn_branch_type_return: + vtn_assert(block); + vtn_emit_ret_store(b, block); + nir_jump(&b->nb, nir_jump_return); + break; + + case vtn_branch_type_discard: + if (b->convert_discard_to_demote) + nir_demote(&b->nb); + else + nir_discard(&b->nb); + break; + + case vtn_branch_type_terminate_invocation: + nir_terminate(&b->nb); + break; + + case vtn_branch_type_ignore_intersection: + nir_ignore_ray_intersection(&b->nb); + nir_jump(&b->nb, nir_jump_halt); + break; + + case vtn_branch_type_terminate_ray: + nir_terminate_ray(&b->nb); + nir_jump(&b->nb, nir_jump_halt); + break; + + case vtn_branch_type_emit_mesh_tasks: { + vtn_assert(block); + vtn_assert(block->branch); + + const uint32_t *w = block->branch; + vtn_assert((w[0] & SpvOpCodeMask) == SpvOpEmitMeshTasksEXT); + + /* Launches mesh shader workgroups from the task shader. + * Arguments are: vec(x, y, z), payload pointer + */ + nir_ssa_def *dimensions = + nir_vec3(&b->nb, vtn_get_nir_ssa(b, w[1]), + vtn_get_nir_ssa(b, w[2]), + vtn_get_nir_ssa(b, w[3])); + + /* The payload variable is optional. + * We don't have a NULL deref in NIR, so just emit the explicit + * intrinsic when there is no payload. + */ + const unsigned count = w[0] >> SpvWordCountShift; + if (count == 4) + nir_launch_mesh_workgroups(&b->nb, dimensions); + else if (count == 5) + nir_launch_mesh_workgroups_with_payload_deref(&b->nb, dimensions, + vtn_get_nir_ssa(b, w[4])); + else + vtn_fail("Invalid EmitMeshTasksEXT."); + + nir_jump(&b->nb, nir_jump_halt); + break; + } + + default: + vtn_fail("Invalid branch type"); + } +} + +static nir_selection_control +vtn_selection_control(struct vtn_builder *b, SpvSelectionControlMask control) +{ + if (control == SpvSelectionControlMaskNone) + return nir_selection_control_none; + else if (control & SpvSelectionControlDontFlattenMask) + return nir_selection_control_dont_flatten; + else if (control & SpvSelectionControlFlattenMask) + return nir_selection_control_flatten; + else + vtn_fail("Invalid selection control"); +} + +static void +vtn_emit_block(struct vtn_builder *b, struct vtn_block *block, + vtn_instruction_handler handler) +{ + const uint32_t *block_start = block->label; + const uint32_t *block_end = block->merge ? block->merge : + block->branch; + + block_start = vtn_foreach_instruction(b, block_start, block_end, + vtn_handle_phis_first_pass); + + vtn_foreach_instruction(b, block_start, block_end, handler); + + block->end_nop = nir_nop(&b->nb); + + if (block->parent->type == vtn_construct_type_switch) { + /* Switch is handled as a sequence of NIR if for each of the cases. */ + + } else if (block->successors_count == 1) { + vtn_assert(block->successors[0].branch_type != vtn_branch_type_none); + vtn_emit_branch(b, block, &block->successors[0]); + + } else if (block->successors_count == 2) { + struct vtn_successor *then_succ = &block->successors[0]; + struct vtn_successor *else_succ = &block->successors[1]; + struct vtn_construct *c = block->parent; + + nir_ssa_def *cond = vtn_get_nir_ssa(b, block->branch[1]); + if (then_succ->block == else_succ->block) + cond = nir_imm_true(&b->nb); + + /* The branches will already be emitted here, so for paths that + * doesn't have blocks inside the construct, e.g. that are an + * exit from the construct, nothing else is needed. + */ + nir_if *sel = nir_push_if(&b->nb, cond); + vtn_emit_branch(b, block, then_succ); + if (then_succ->block != else_succ->block) { + nir_push_else(&b->nb, NULL); + vtn_emit_branch(b, block, else_succ); + } + nir_pop_if(&b->nb, NULL); + + if (c->type == vtn_construct_type_selection && + block->pos == c->start_pos) { + /* This is the start of a selection construct. Record the nir_if in + * the construct so we can close it properly and handle the then and + * else cases in block iteration. + */ + vtn_assert(c->nif == NULL); + c->nif = sel; + + vtn_assert(block->merge != NULL); + + SpvOp merge_op = block->merge[0] & SpvOpCodeMask; + if (merge_op == SpvOpSelectionMerge) + sel->control = vtn_selection_control(b, block->merge[2]); + + /* In most cases, vtn_emit_cf_func_structured() will place the cursor + * in the correct side of the nir_if. However, in the case where the + * selection construct is empty, we need to ensure that the cursor is + * at least inside the nir_if or NIR will assert when we try to close + * it with nir_pop_if(). + */ + b->nb.cursor = nir_before_cf_list(&sel->then_list); + } else { + vtn_fail_if(then_succ->branch_type == vtn_branch_type_forward && + else_succ->branch_type == vtn_branch_type_forward && + then_succ->block != else_succ->block, + "An OpSelectionMerge instruction is required to precede " + "an OpBranchConditional instruction that has different " + "True Label and False Label operands where neither are " + "declared merge blocks or Continue Targets."); + + if (then_succ->branch_type == vtn_branch_type_forward) { + b->nb.cursor = nir_before_cf_list(&sel->then_list); + } else if (else_succ->branch_type == vtn_branch_type_forward) { + b->nb.cursor = nir_before_cf_list(&sel->else_list); + } else { + /* Leave it alone */ + } + } + } +} + +static nir_ssa_def * +vtn_switch_case_condition(struct vtn_builder *b, struct vtn_construct *swtch, + nir_ssa_def *sel, struct vtn_case *cse) +{ + vtn_assert(swtch->type == vtn_construct_type_switch); + + if (cse->is_default) { + nir_ssa_def *any = nir_imm_false(&b->nb); + + struct vtn_block *header = b->func->ordered_blocks[swtch->start_pos]; + + for (unsigned j = 0; j < header->successors_count; j++) { + struct vtn_successor *succ = &header->successors[j]; + struct vtn_case *other = succ->block->switch_case; + + if (other->is_default) + continue; + any = nir_ior(&b->nb, any, + vtn_switch_case_condition(b, swtch, sel, other)); + } + + return nir_inot(&b->nb, any); + } else { + nir_ssa_def *cond = nir_imm_false(&b->nb); + util_dynarray_foreach(&cse->values, uint64_t, val) + cond = nir_ior(&b->nb, cond, nir_ieq_imm(&b->nb, sel, *val)); + return cond; + } +} + +static nir_loop_control +vtn_loop_control(struct vtn_builder *b, SpvLoopControlMask control) +{ + if (control == SpvLoopControlMaskNone) + return nir_loop_control_none; + else if (control & SpvLoopControlDontUnrollMask) + return nir_loop_control_dont_unroll; + else if (control & SpvLoopControlUnrollMask) + return nir_loop_control_unroll; + else if ((control & SpvLoopControlDependencyInfiniteMask) || + (control & SpvLoopControlDependencyLengthMask) || + (control & SpvLoopControlMinIterationsMask) || + (control & SpvLoopControlMaxIterationsMask) || + (control & SpvLoopControlIterationMultipleMask) || + (control & SpvLoopControlPeelCountMask) || + (control & SpvLoopControlPartialCountMask)) { + /* We do not do anything special with these yet. */ + return nir_loop_control_none; + } else { + vtn_fail("Invalid loop control"); + } +} + +static void +vtn_emit_control_flow_propagation(struct vtn_builder *b, + struct vtn_construct *top) +{ + if (top->type == vtn_construct_type_function || + top->type == vtn_construct_type_continue || + top->type == vtn_construct_type_switch) + return; + + /* Find the innermost parent with a NIR loop. */ + struct vtn_construct *parent_with_nloop = NULL; + for (struct vtn_construct *c = top->parent; c; c = c->parent) { + if (c->nloop) { + parent_with_nloop = c; + break; + } + } + if (parent_with_nloop == NULL) + return; + + /* If there's another nloop in the parent chain, decide whether we need + * to emit conditional continue/break after top construct is closed. + */ + + if (top->needs_continue_propagation && + parent_with_nloop == top->innermost_loop) { + struct vtn_construct *loop = top->innermost_loop; + vtn_assert(loop); + vtn_assert(loop != top); + + nir_push_if(&b->nb, nir_load_var(&b->nb, loop->continue_var)); + nir_jump(&b->nb, nir_jump_continue); + nir_pop_if(&b->nb, NULL); + } + + if (top->needs_break_propagation) { + vtn_assert(parent_with_nloop->break_var); + nir_push_if(&b->nb, nir_load_var(&b->nb, parent_with_nloop->break_var)); + nir_jump(&b->nb, nir_jump_break); + nir_pop_if(&b->nb, NULL); + } +} + +static inline nir_variable * +vtn_create_local_bool(struct vtn_builder *b, const char *name) +{ + return nir_local_variable_create(b->nb.impl, glsl_bool_type(), name); +} + +void +vtn_emit_cf_func_structured(struct vtn_builder *b, struct vtn_function *func, + vtn_instruction_handler handler) +{ + struct vtn_construct *current = + list_first_entry(&func->constructs, struct vtn_construct, link); + vtn_assert(current->type == vtn_construct_type_function); + + /* Walk the blocks in order keeping track of the constructs that started + * but haven't ended yet. When constructs start and end, add extra code to + * setup the NIR control flow (different for each construct), also add + * extra code for propagating certain branch types. + */ + + struct vtn_construct_stack stack; + init_construct_stack(&stack, b); + push_construct(&stack, current); + + for (unsigned i = 0; i < func->ordered_blocks_count; i++) { + struct vtn_block *block = func->ordered_blocks[i]; + struct vtn_construct *top = top_construct(&stack); + + /* Close out any past constructs and make sure the cursor is at the + * right place to start this block. For each block, there are three + * cases we care about here: + * + * 1. It is the block at the end (in our reverse structured post-order + * traversal) of one or more constructs and closes them. + * + * 2. It is an early merge of a selection construct. + * + * 3. It is the start of the then or else case of a selection construct + * and we may have previously been emitting code in the other side. + */ + + /* Close (or early merge) any constructs that end at this block. */ + bool merged_any_constructs = false; + while (top->end_pos == block->pos || top->merge_pos == block->pos) { + merged_any_constructs = true; + if (top->nif) { + const bool has_early_merge = top->merge_pos != top->end_pos; + + if (!has_early_merge) { + nir_pop_if(&b->nb, top->nif); + } else if (block->pos == top->merge_pos) { + /* This is an early merge. */ + + nir_pop_if(&b->nb, top->nif); + + /* The extra dummy "if (true)" for the merged part avoids + * generating multiple jumps in sequence and upsetting + * NIR rules. We'll pop it in the case below when we reach + * the end_pos block. + */ + nir_push_if(&b->nb, nir_imm_true(&b->nb)); + + /* Stop since this construct still has more blocks. */ + break; + } else { + /* Pop the dummy if added for the blocks after the early merge. */ + vtn_assert(block->pos == top->end_pos); + nir_pop_if(&b->nb, NULL); + } + } + + if (top->nloop) { + /* For constructs that are not SPIR-V loop, a NIR loop may be used + * to provide richer control flow. So we add a nir break to cause + * the loop stop at the first iteration, unless there's already a + * jump at the end of the last block. + */ + if (top->type != vtn_construct_type_loop) { + nir_block *last = nir_loop_last_block(top->nloop); + if (!nir_block_ends_in_jump(last)) { + b->nb.cursor = nir_after_block(last); + nir_jump(&b->nb, nir_jump_break); + } + } + + nir_pop_loop(&b->nb, top->nloop); + } + + vtn_emit_control_flow_propagation(b, top); + + pop_construct(&stack); + top = top_construct(&stack); + } + + /* We are fully inside the current top. */ + vtn_assert(block->pos < top->end_pos); + + /* Move the cursor to the right side of a selection construct. + * + * If we merged any constructs, we don't need to move because + * either: this is an early merge and we already set the cursor above; + * or a construct ended, and this is a 'merge block' for that + * construct, so it can't also be a 'Target' for an outer conditional. + */ + if (!merged_any_constructs && top->type == vtn_construct_type_selection && + (block->pos == top->then_pos || block->pos == top->else_pos)) { + vtn_assert(top->nif); + + struct vtn_block *header = func->ordered_blocks[top->start_pos]; + vtn_assert(header->successors_count == 2); + + if (block->pos == top->then_pos) + b->nb.cursor = nir_before_cf_list(&top->nif->then_list); + else + b->nb.cursor = nir_before_cf_list(&top->nif->else_list); + } + + /* Open any constructs which start at this block. + * + * Constructs which are designated by Op*Merge are considered to start + * at the block which contains the merge instruction. This means that + * loops constructs start at the first block inside the loop while + * selection and switch constructs start at the block containing the + * OpBranchConditional or OpSwitch. + */ + while (current->link.next != &func->constructs) { + struct vtn_construct *next = + list_entry(current->link.next, struct vtn_construct, link); + + /* Stop once we find a construct that doesn't start in this block. */ + if (next->start_pos != block->pos) + break; + + switch (next->type) { + case vtn_construct_type_function: + unreachable("should've already entered function construct"); + break; + + case vtn_construct_type_selection: { + /* Add the wrapper loop now and the nir_if, along the contents of + * this entire block, will get added inside the loop as part of + * vtn_emit_block() below. + */ + if (next->needs_nloop) { + next->break_var = vtn_create_local_bool(b, "if_break"); + nir_store_var(&b->nb, next->break_var, nir_imm_false(&b->nb), 1); + next->nloop = nir_push_loop(&b->nb); + } + break; + } + + case vtn_construct_type_loop: { + next->break_var = vtn_create_local_bool(b, "loop_break"); + next->continue_var = vtn_create_local_bool(b, "loop_continue"); + + nir_store_var(&b->nb, next->break_var, nir_imm_false(&b->nb), 1); + next->nloop = nir_push_loop(&b->nb); + nir_store_var(&b->nb, next->continue_var, nir_imm_false(&b->nb), 1); + + next->nloop->control = vtn_loop_control(b, block->merge[3]); + + break; + } + + case vtn_construct_type_continue: { + struct vtn_construct *loop = next->parent; + assert(loop->type == vtn_construct_type_loop); + assert(!vtn_is_single_block_loop(loop)); + + nir_push_continue(&b->nb, loop->nloop); + + break; + } + + case vtn_construct_type_switch: { + /* Switch is not translated to any NIR node, all is handled by + * each individual case construct. + */ + for (unsigned j = 0; j < block->successors_count; j++) { + struct vtn_successor *s = &block->successors[j]; + if (s->block && s->block->pos < next->end_pos) { + struct vtn_construct *c = s->block->parent->innermost_case; + vtn_assert(c->type == vtn_construct_type_case); + if (c->needs_fallthrough) { + c->fallthrough_var = vtn_create_local_bool(b, "fallthrough"); + nir_store_var(&b->nb, c->fallthrough_var, nir_imm_false(&b->nb), 1); + } + } + } + break; + } + + case vtn_construct_type_case: { + struct vtn_construct *swtch = next->parent; + struct vtn_block *header = func->ordered_blocks[swtch->start_pos]; + + nir_ssa_def *sel = vtn_get_nir_ssa(b, header->branch[1]); + nir_ssa_def *case_condition = + vtn_switch_case_condition(b, swtch, sel, block->switch_case); + if (next->fallthrough_var) { + case_condition = + nir_ior(&b->nb, case_condition, + nir_load_var(&b->nb, next->fallthrough_var)); + } + + if (next->needs_nloop) { + next->break_var = vtn_create_local_bool(b, "case_break"); + nir_store_var(&b->nb, next->break_var, nir_imm_false(&b->nb), 1); + next->nloop = nir_push_loop(&b->nb); + } + + next->nif = nir_push_if(&b->nb, case_condition); + + break; + } + } + + current = next; + push_construct(&stack, next); + } + + vtn_emit_block(b, block, handler); + } + + vtn_assert(count_construct_stack(&stack) == 1); +} diff --git a/src/microsoft/ci/spirv2dxil_reference.txt b/src/microsoft/ci/spirv2dxil_reference.txt index 5e4fe239e42..5dbaf654f4d 100644 --- a/src/microsoft/ci/spirv2dxil_reference.txt +++ b/src/microsoft/ci/spirv2dxil_reference.txt @@ -490,13 +490,7 @@ Test:SpvParserCFGTest_ClassifyCFGEdges_LoopContinue_FromNestedSwitchDefaultBody_ Test:SpvParserCFGTest_ClassifyCFGEdges_LoopContinue_FromNestedSwitchDefaultBody_Unconditional.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_ClassifyCFGEdges_LoopContinue_LoopBodyToContinue.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_ClassifyCFGEdges_Pathological_Forward_LoopHeadSplitBody.spvasm:main|Fragment: Pass -Test:SpvParserCFGTest_ClassifyCFGEdges_Pathological_Forward_Premerge.spvasm:main|Fragment: Fail -SPIR-V parsing FAILED: - - Invalid back or cross-edge in the CFG - 0 bytes into the SPIR-V binary - - +Test:SpvParserCFGTest_ClassifyCFGEdges_Pathological_Forward_Premerge.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_ClassifyCFGEdges_Pathological_Forward_Regardless.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_ClassifyCFGEdges_SwitchBreak_FromNestedIf_Conditional.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_ClassifyCFGEdges_SwitchBreak_FromNestedIf_Unconditional.spvasm:main|Fragment: Pass @@ -536,13 +530,7 @@ Test:SpvParserCFGTest_ComputeBlockOrder_Nest_If_In_SwitchCase.spvasm:main|Fragme Test:SpvParserCFGTest_ComputeBlockOrder_Nest_IfBreak_In_SwitchCase.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_ComputeBlockOrder_Nest_IfFallthrough_In_SwitchCase.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_ComputeBlockOrder_OneBlock.spvasm:main|Fragment: Pass -Test:SpvParserCFGTest_ComputeBlockOrder_ReorderSequence.spvasm:main|Fragment: Fail -SPIR-V parsing FAILED: - - Invalid back or cross-edge in the CFG - 0 bytes into the SPIR-V binary - - +Test:SpvParserCFGTest_ComputeBlockOrder_ReorderSequence.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_ComputeBlockOrder_RespectConditionalBranchOrder.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_ComputeBlockOrder_RespectSwitchCaseFallthrough.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_ComputeBlockOrder_RespectSwitchCaseFallthrough_FromCaseToDefaultToCase.spvasm:main|Fragment: Pass @@ -602,41 +590,17 @@ Test:SpvParserCFGTest_EmitBody_BranchConditional_SwitchBreak_SwitchBreak_NotLast Test:SpvParserCFGTest_EmitBody_FalseBranch_LoopBreak.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_EmitBody_FalseBranch_LoopContinue.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_EmitBody_FalseBranch_SwitchBreak.spvasm:main|Fragment: Pass -Test:SpvParserCFGTest_EmitBody_If_Else_Premerge.spvasm:main|Fragment: Fail -SPIR-V parsing FAILED: - - Invalid back or cross-edge in the CFG - 0 bytes into the SPIR-V binary - - +Test:SpvParserCFGTest_EmitBody_If_Else_Premerge.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_EmitBody_If_Empty.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_EmitBody_If_Nest_If.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_EmitBody_If_NoThen_Else.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_EmitBody_If_Then_Else.spvasm:main|Fragment: Pass -Test:SpvParserCFGTest_EmitBody_If_Then_Else_Premerge.spvasm:main|Fragment: Fail -SPIR-V parsing FAILED: - - Invalid back or cross-edge in the CFG - 0 bytes into the SPIR-V binary - - +Test:SpvParserCFGTest_EmitBody_If_Then_Else_Premerge.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_EmitBody_If_Then_NoElse.spvasm:main|Fragment: Pass -Test:SpvParserCFGTest_EmitBody_If_Then_Premerge.spvasm:main|Fragment: Fail -SPIR-V parsing FAILED: - - Invalid back or cross-edge in the CFG - 0 bytes into the SPIR-V binary - - +Test:SpvParserCFGTest_EmitBody_If_Then_Premerge.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_EmitBody_IfBreak_FromElse_ForwardWithinElse.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_EmitBody_IfBreak_FromThen_ForwardWithinThen.spvasm:main|Fragment: Pass -Test:SpvParserCFGTest_EmitBody_IfBreak_FromThenWithForward_FromElseWithForward_AlsoPremerge.spvasm:main|Fragment: Fail -SPIR-V parsing FAILED: - - Invalid back or cross-edge in the CFG - 0 bytes into the SPIR-V binary - - +Test:SpvParserCFGTest_EmitBody_IfBreak_FromThenWithForward_FromElseWithForward_AlsoPremerge.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_EmitBody_IfSelection_TrueBranch_LoopBreak.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_EmitBody_Kill_InsideIf.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_EmitBody_Kill_InsideLoop.spvasm:main|Fragment: Pass @@ -685,27 +649,9 @@ Test:SpvParserCFGTest_FindIfSelectionInternalHeaders_FalseBranch_LoopContinue_Ok Test:SpvParserCFGTest_FindIfSelectionInternalHeaders_FalseBranch_SwitchBreak_Ok.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_FindIfSelectionInternalHeaders_IfOnly.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_FindIfSelectionInternalHeaders_NoIf.spvasm:main|Fragment: Pass -Test:SpvParserCFGTest_FindIfSelectionInternalHeaders_Premerge_ElseDirectToThen.spvasm:main|Fragment: Fail -SPIR-V parsing FAILED: - - Invalid back or cross-edge in the CFG - 0 bytes into the SPIR-V binary - - -Test:SpvParserCFGTest_FindIfSelectionInternalHeaders_Premerge_Simple.spvasm:main|Fragment: Fail -SPIR-V parsing FAILED: - - Invalid back or cross-edge in the CFG - 0 bytes into the SPIR-V binary - - -Test:SpvParserCFGTest_FindIfSelectionInternalHeaders_Premerge_ThenDirectToElse.spvasm:main|Fragment: Fail -SPIR-V parsing FAILED: - - Invalid back or cross-edge in the CFG - 0 bytes into the SPIR-V binary - - +Test:SpvParserCFGTest_FindIfSelectionInternalHeaders_Premerge_ElseDirectToThen.spvasm:main|Fragment: Pass +Test:SpvParserCFGTest_FindIfSelectionInternalHeaders_Premerge_Simple.spvasm:main|Fragment: Pass +Test:SpvParserCFGTest_FindIfSelectionInternalHeaders_Premerge_ThenDirectToElse.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_FindIfSelectionInternalHeaders_Regardless.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_FindIfSelectionInternalHeaders_ThenElse.spvasm:main|Fragment: Pass Test:SpvParserCFGTest_FindIfSelectionInternalHeaders_TrueBranch_LoopBreak_Ok.spvasm:main|Fragment: Pass