util,vulkan,mesa,compiler: Generate source files with utf8 encoding from mako template

This is for fixes the following error:
FAILED: src/vulkan/runtime/vk_synchronization_helpers.c src/vulkan/runtime/vk_synchronization_helpers.h
"C:\CI-Tools\msys64\mingw64\bin/python3.EXE" "../../src/vulkan/util/vk_synchronization_helpers_gen.py" "--xml" "../../src/vulkan/registry/vk.xml" "--out-c" "src/vulkan/runtime/vk_synchronization_helpers.c" "--beta" "false"

Traceback (most recent call last):
  File "C:/work/xemu/mesa/src/vulkan/util/vk_synchronization_helpers_gen.py", line 213, in main
    f.write(TEMPLATE_C.render(**environment))
UnicodeEncodeError: 'gbk' codec can't encode character '\xa9' in position 15: illegal multibyte sequence

Signed-off-by: Yonggang Luo <luoyonggang@gmail.com>
Acked-by: Eric Engestrom <eric@igalia.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/26515>

src/compiler/builtin_types_c.py

@@ -56,5 +56,5 @@ for t in BUILTIN_TYPES:
     t["name_id"] = id
     id += len(name) + 1
 
-with open(output, 'w') as f:
+with open(output, 'w', encoding='utf-8') as f:
     f.write(Template(template).render(BUILTIN_TYPES=BUILTIN_TYPES, NAME_ARRAY=NAME_ARRAY))

src/compiler/builtin_types_cpp_h.py

@@ -33,5 +33,5 @@ if len(sys.argv) < 2:
 
 output = sys.argv[1]
 
-with open(output, 'w') as f:
+with open(output, 'w', encoding='utf-8') as f:
     f.write(Template(template).render(BUILTIN_TYPES=BUILTIN_TYPES))

src/compiler/builtin_types_h.py

@@ -29,5 +29,5 @@ if len(sys.argv) < 2:
 
 output = sys.argv[1]
 
-with open(output, 'w') as f:
+with open(output, 'w', encoding='utf-8') as f:
     f.write(Template(template).render(BUILTIN_TYPES=BUILTIN_TYPES))

src/compiler/isaspec/decode.py

@@ -392,11 +392,11 @@ def main():
     s = State(isa)
 
     try:
-        with open(args.out_c, 'w') as f:
+        with open(args.out_c, 'w', encoding='utf-8') as f:
             out_h_basename = os.path.basename(args.out_h)
             f.write(Template(template).render(isa=isa, s=s, header=out_h_basename))
 
-        with open(args.out_h, 'w') as f:
+        with open(args.out_h, 'w', encoding='utf-8') as f:
             f.write(Template(header).render(isa=isa, guard=guard(args.out_h)))
 
     except Exception:

src/compiler/isaspec/encode.py

@@ -704,7 +704,7 @@ def main():
     s = State(isa)
 
     try:
-        with open(args.out_h, 'w') as f:
+        with open(args.out_h, 'w', encoding='utf-8') as f:
             encode_bitset = Template(encode_bitset_template)
             f.write(Template(template).render(s=s, encode_bitset=encode_bitset))
 

src/compiler/nir/nir_intrinsics_c.py

@@ -78,7 +78,7 @@ def main():
     args = parser.parse_args()
 
     path = os.path.join(args.outdir, 'nir_intrinsics.c')
-    with open(path, 'w') as f:
+    with open(path, 'w', encoding='utf-8') as f:
         f.write(Template(template).render(
             INTR_OPCODES=INTR_OPCODES, INTR_INDICES=INTR_INDICES,
             reduce=reduce, operator=operator))

src/compiler/nir/nir_intrinsics_h.py

@@ -61,7 +61,7 @@ def main():
     args = parser.parse_args()
 
     path = os.path.join(args.outdir, 'nir_intrinsics.h')
-    with open(path, 'w') as f:
+    with open(path, 'w', encoding='utf-8') as f:
         f.write(Template(template).render(INTR_OPCODES=INTR_OPCODES, INTR_INDICES=INTR_INDICES))
 
 if __name__ == '__main__':

src/compiler/nir/nir_intrinsics_indices_h.py

@@ -88,7 +88,7 @@ def main():
     args = parser.parse_args()
 
     path = os.path.join(args.outdir, 'nir_intrinsics_indices.h')
-    with open(path, 'w') as f:
+    with open(path, 'w', encoding='utf-8') as f:
         f.write(Template(template).render(INTR_INDICES=INTR_INDICES))
 
 if __name__ == '__main__':

src/compiler/spirv/spirv_info_c.py

@@ -128,5 +128,5 @@ if __name__ == "__main__":
         collect_opcodes(spirv_info),
     ]
 
-    with open(pargs.out, 'w') as f:
+    with open(pargs.out, 'w', encoding='utf-8') as f:
         f.write(TEMPLATE.render(info=info))

src/compiler/spirv/vtn_gather_types_c.py

@@ -115,7 +115,7 @@ if __name__ == "__main__":
     opcodes = list(find_result_types(spirv_info))
 
     try:
-        with open(args.out, 'w') as f:
+        with open(args.out, 'w', encoding='utf-8') as f:
             f.write(TEMPLATE.render(opcodes=opcodes))
     except Exception:
         # In the even there's an error this imports some helpers from mako

src/compiler/spirv/vtn_generator_ids_h.py

@@ -59,5 +59,5 @@ if __name__ == "__main__":
     tree = ET.parse(pargs.xml)
     root = tree.getroot()
 
-    with open(pargs.out, 'w') as f:
+    with open(pargs.out, 'w', encoding='utf-8') as f:
         f.write(TEMPLATE.render(root=root))

src/mesa/main/format_fallback.py

@@ -129,7 +129,7 @@ def main():
         'intensity_to_red_map': list(get_intensity_to_red_map(formats)),
     }
 
-    with open(pargs.out, 'w') as f:
+    with open(pargs.out, 'w', encoding='utf-8') as f:
         f.write(TEMPLATE.render(**template_env))
 
 if __name__ == "__main__":

src/util/driconf_static.py

@@ -237,6 +237,6 @@ args = parser.parse_args()
 xml = args.drirc
 dst = args.header
 
-with open(dst, 'wb') as f:
-    f.write(Template(template, output_encoding='utf-8').render(driconf=DriConf(xml)))
+with open(dst, 'w', encoding='utf-8') as f:
+    f.write(Template(template).render(driconf=DriConf(xml)))
 

src/util/perf/u_trace.py

@@ -511,7 +511,7 @@ def utrace_generate(cpath, hpath, ctx_param, trace_toggle_name=None,
     """
     if cpath is not None:
         hdr = os.path.basename(cpath).rsplit('.', 1)[0] + '.h'
-        with open(cpath, 'w') as f:
+        with open(cpath, 'w', encoding='utf-8') as f:
             f.write(Template(src_template).render(
                 hdr=hdr,
                 ctx_param=ctx_param,
@@ -523,7 +523,7 @@ def utrace_generate(cpath, hpath, ctx_param, trace_toggle_name=None,
 
     if hpath is not None:
         hdr = os.path.basename(hpath)
-        with open(hpath, 'w') as f:
+        with open(hpath, 'w', encoding='utf-8') as f:
             f.write(Template(hdr_template).render(
                 hdrname=hdr.rstrip('.h').upper(),
                 ctx_param=ctx_param,
@@ -607,8 +607,8 @@ trace_payload_as_extra_${trace_name}(perfetto::protos::pbzero::GpuRenderStageEve
 def utrace_generate_perfetto_utils(hpath,basename="tracepoint"):
     if hpath is not None:
         hdr = os.path.basename(hpath)
-        with open(hpath, 'wb') as f:
-            f.write(Template(perfetto_utils_hdr_template, output_encoding='utf-8').render(
+        with open(hpath, 'w', encoding='utf-8') as f:
+            f.write(Template(perfetto_utils_hdr_template).render(
                 basename=basename,
                 hdrname=hdr.rstrip('.h').upper(),
                 HEADERS=[h for h in HEADERS if h.scope & HeaderScope.PERFETTO],

src/vulkan/runtime/vk_format_info_gen.py

@@ -65,7 +65,7 @@ vk_format_get_class_info(VkFormat format);
 #endif
 
 #endif
-""", output_encoding='utf-8')
+""")
 
 TEMPLATE_C = Template(COPYRIGHT + """
 /* This file generated from ${filename}, don't edit directly. */
@@ -144,7 +144,7 @@ vk_format_get_class_info(VkFormat format)
     const struct vk_format_info *format_info = vk_format_get_info(format);
     return &class_infos[format_info->class];
 }
-""", output_encoding='utf-8')
+""")
 
 def to_enum_name(prefix, name):
     return "%s" % prefix + re.sub('([^A-Za-z0-9_])', '_', name).upper()
@@ -226,10 +226,10 @@ def main():
     }
 
     try:
-        with open(args.out_h, 'wb') as f:
+        with open(args.out_h, 'w', encoding='utf-8') as f:
             guard = os.path.basename(args.out_h).replace('.', '_').upper()
             f.write(TEMPLATE_H.render(guard=guard, **environment))
-        with open(args.out_c, 'wb') as f:
+        with open(args.out_c, 'w', encoding='utf-8') as f:
             f.write(TEMPLATE_C.render(**environment))
     except Exception:
         # In the event there's an error, this imports some helpers from mako

src/vulkan/util/vk_cmd_queue_gen.py

@@ -198,7 +198,7 @@ void vk_cmd_queue_execute(struct vk_cmd_queue *queue,
 #ifdef __cplusplus
 }
 #endif
-""", output_encoding='utf-8')
+""")
 
 TEMPLATE_C = Template(COPYRIGHT + """
 /* This file generated from ${filename}, don't edit directly. */
@@ -415,7 +415,7 @@ vk_cmd_enqueue_unless_primary_${c.name}(${c.decl_params()})
 #endif // ${c.guard}
 % endif
 % endfor
-""", output_encoding='utf-8')
+""")
 
 def remove_prefix(text, prefix):
     if text.startswith(prefix):
@@ -664,10 +664,10 @@ def main():
     }
 
     try:
-        with open(args.out_h, 'wb') as f:
+        with open(args.out_h, 'w', encoding='utf-8') as f:
             guard = os.path.basename(args.out_h).replace('.', '_').upper()
             f.write(TEMPLATE_H.render(guard=guard, **environment))
-        with open(args.out_c, 'wb') as f:
+        with open(args.out_c, 'w', encoding='utf-8') as f:
             f.write(TEMPLATE_C.render(**environment))
     except Exception:
         # In the event there's an error, this imports some helpers from mako

src/vulkan/util/vk_dispatch_table_gen.py

@@ -725,13 +725,13 @@ def main():
     # per entry point.
     try:
         if args.out_h:
-            with open(args.out_h, 'w') as f:
+            with open(args.out_h, 'w', encoding='utf-8') as f:
                 f.write(TEMPLATE_H.render(instance_entrypoints=instance_entrypoints,
                                           physical_device_entrypoints=physical_device_entrypoints,
                                           device_entrypoints=device_entrypoints,
                                           filename=os.path.basename(__file__)))
         if args.out_c:
-            with open(args.out_c, 'w') as f:
+            with open(args.out_c, 'w', encoding='utf-8') as f:
                 f.write(TEMPLATE_C.render(instance_entrypoints=instance_entrypoints,
                                           physical_device_entrypoints=physical_device_entrypoints,
                                           device_entrypoints=device_entrypoints,

src/vulkan/util/vk_dispatch_trampolines_gen.py

@@ -169,11 +169,11 @@ def main():
     # per entry point.
     try:
         if args.out_h:
-            with open(args.out_h, 'w') as f:
+            with open(args.out_h, 'w', encoding='utf-8') as f:
                 f.write(TEMPLATE_H.render(entrypoints=entrypoints,
                                           filename=os.path.basename(__file__)))
         if args.out_c:
-            with open(args.out_c, 'w') as f:
+            with open(args.out_c, 'w', encoding='utf-8') as f:
                 f.write(TEMPLATE_C.render(entrypoints=entrypoints,
                                           filename=os.path.basename(__file__)))
     except Exception:

src/vulkan/util/vk_extensions_gen.py

@@ -219,11 +219,11 @@ def gen_extensions(driver, xml_files, api_versions, max_api_version,
     }
 
     if out_h:
-        with open(out_h, 'w') as f:
+        with open(out_h, 'w', encoding='utf-8') as f:
             f.write(_TEMPLATE_H.render(**template_env))
 
     if out_c:
-        with open(out_c, 'w') as f:
+        with open(out_c, 'w', encoding='utf-8') as f:
             f.write(_TEMPLATE_C.render(**template_env))
 
 

src/vulkan/util/vk_icd_gen.py

@@ -78,7 +78,7 @@ if __name__ == '__main__':
     }
 
     if args.out:
-        with open(args.out, 'w') as f:
+        with open(args.out, 'w', encoding='utf-8') as f:
             json.dump(json_data, f, **json_params)
     else:
         print(json.dumps(json_data, **json_params))

src/vulkan/util/vk_physical_device_features_gen.py

@@ -156,7 +156,7 @@ vk_set_physical_device_features_1_0(struct vk_features *all_features,
 #endif
 
 #endif
-""", output_encoding='utf-8')
+""")
 
 TEMPLATE_C = Template(COPYRIGHT + """
 /* This file generated from ${filename}, don't edit directly. */
@@ -335,7 +335,7 @@ vk_set_physical_device_features_1_0(struct vk_features *all_features,
       all_features->${flag} = true;
 % endfor
 }
-""", output_encoding='utf-8')
+""")
 
 def get_pdev_features(doc):
     _type = doc.find(".types/type[@name='VkPhysicalDeviceFeatures']")
@@ -457,9 +457,9 @@ def main():
     }
 
     try:
-        with open(args.out_c, 'wb') as f:
+        with open(args.out_c, 'w', encoding='utf-8') as f:
             f.write(TEMPLATE_C.render(**environment))
-        with open(args.out_h, 'wb') as f:
+        with open(args.out_h, 'w', encoding='utf-8') as f:
             f.write(TEMPLATE_H.render(**environment))
     except Exception:
         # In the event there's an error, this uses some helpers from mako

src/vulkan/util/vk_physical_device_properties_gen.py

@@ -120,7 +120,7 @@ struct vk_properties {
 #endif
 
 #endif
-""", output_encoding="utf-8")
+""")
 
 TEMPLATE_C = Template(COPYRIGHT + """
 /* This file generated from ${filename}, don"t edit directly. */
@@ -190,7 +190,7 @@ vk_common_GetPhysicalDeviceProperties2(VkPhysicalDevice physicalDevice,
       }
    }
 }
-""", output_encoding="utf-8")
+""")
 
 def get_pdev_properties(doc, struct_name):
     _type = doc.find(".types/type[@name=\"VkPhysicalDevice%s\"]" % struct_name)
@@ -316,9 +316,9 @@ def main():
     }
 
     try:
-        with open(args.out_c, "wb") as f:
+        with open(args.out_c, "w", encoding='utf-8') as f:
             f.write(TEMPLATE_C.render(**environment))
-        with open(args.out_h, "wb") as f:
+        with open(args.out_h, "w", encoding='utf-8') as f:
             f.write(TEMPLATE_H.render(**environment))
     except Exception:
         # In the event there"s an error, this uses some helpers from mako

src/vulkan/util/vk_synchronization_helpers_gen.py

@@ -209,7 +209,7 @@ def main():
     }
 
     try:
-        with open(args.out_c, 'w') as f:
+        with open(args.out_c, 'w', encoding='utf-8') as f:
             f.write(TEMPLATE_C.render(**environment))
     except Exception:
         # In the event there's an error, this imports some helpers from mako