From 8c51980332ed1bbc0f7ff5e727db3c9e51d30825 Mon Sep 17 00:00:00 2001 From: Markus Armbruster Date: Fri, 9 Mar 2018 08:12:41 -0500 Subject: [PATCH] qapi-gen: New common driver for code and doc generators Whenever qapi-schema.json changes, we run six programs eleven times to update eleven files. Similar for qga/qapi-schema.json. This is silly. Replace the six programs by a single program that spits out all eleven files. The programs become modules in new Python package qapi, along with the helper library. This requires moving them to scripts/qapi/. While moving them, consistently drop executable mode bits. Backports commit fb0bc835e56b894cbc7236294921e5393c786ad8 from qemu --- .gitignore | 1 + msvc/unicorn/qapi-types.c | 21 ++++++--- msvc/unicorn/qapi-types.h | 20 ++++++--- msvc/unicorn/qapi-visit.c | 38 +++++++++++++--- msvc/unicorn/qapi-visit.h | 11 ++--- qemu/Makefile | 45 ++++++++++--------- qemu/Makefile.target | 4 +- qemu/scripts/qapi-gen.py | 43 ++++++++++++++++++ qemu/scripts/qapi/__init__.py | 0 qemu/scripts/{qapi.py => qapi/common.py} | 18 ++------ .../scripts/{qapi-event.py => qapi/events.py} | 19 ++------ qemu/scripts/{qapi-types.py => qapi/types.py} | 30 ++----------- qemu/scripts/{qapi-visit.py => qapi/visit.py} | 30 ++----------- 13 files changed, 151 insertions(+), 129 deletions(-) create mode 100644 qemu/scripts/qapi-gen.py create mode 100644 qemu/scripts/qapi/__init__.py rename qemu/scripts/{qapi.py => qapi/common.py} (99%) rename qemu/scripts/{qapi-event.py => qapi/events.py} (93%) rename qemu/scripts/{qapi-types.py => qapi/types.py} (90%) rename qemu/scripts/{qapi-visit.py => qapi/visit.py} (92%) diff --git a/.gitignore b/.gitignore index b68a0c80..809e8e47 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ qemu/qapi-types.h qemu/qapi-visit.h qemu/qapi-types.c qemu/qapi-visit.c +qemu/qapi-gen-timestamp tags qemu/config-host.ld qemu/config.log diff --git a/msvc/unicorn/qapi-types.c b/msvc/unicorn/qapi-types.c index e69ffc36..ea58db39 100644 --- a/msvc/unicorn/qapi-types.c +++ b/msvc/unicorn/qapi-types.c @@ -1,17 +1,13 @@ /* AUTOMATICALLY GENERATED, DO NOT MODIFY */ /* - * deallocation functions for schema-defined QAPI types + * Schema-defined QAPI types * * Copyright IBM, Corp. 2011 - * - * Authors: - * Anthony Liguori - * Michael Roth + * Copyright (c) 2013-2018 Red Hat Inc. * * This work is licensed under the terms of the GNU LGPL, version 2.1 or later. * See the COPYING.LIB file in the top-level directory. - * */ #include "qemu/osdep.h" @@ -182,6 +178,19 @@ void qapi_free_intList(intList *obj) visit_free(v); } +void qapi_free_nullList(nullList *obj) +{ + Visitor *v; + + if (!obj) { + return; + } + + v = qapi_dealloc_visitor_new(); + visit_type_nullList(v, NULL, &obj, NULL); + visit_free(v); +} + void qapi_free_numberList(numberList *obj) { Visitor *v; diff --git a/msvc/unicorn/qapi-types.h b/msvc/unicorn/qapi-types.h index 9b387e51..c9ea4ec1 100644 --- a/msvc/unicorn/qapi-types.h +++ b/msvc/unicorn/qapi-types.h @@ -1,21 +1,19 @@ /* AUTOMATICALLY GENERATED, DO NOT MODIFY */ /* - * schema-defined QAPI types + * Schema-defined QAPI types * * Copyright IBM, Corp. 2011 - * - * Authors: - * Anthony Liguori + * Copyright (c) 2013-2018 Red Hat Inc. * * This work is licensed under the terms of the GNU LGPL, version 2.1 or later. * See the COPYING.LIB file in the top-level directory. - * */ #ifndef QAPI_TYPES_H #define QAPI_TYPES_H +/* #include "qapi/util.h" */ #ifndef QAPI_TYPES_BUILTIN #define QAPI_TYPES_BUILTIN @@ -97,6 +95,15 @@ struct intList { void qapi_free_intList(intList *obj); +typedef struct nullList nullList; + +struct nullList { + nullList *next; + QNull *value; +}; + +void qapi_free_nullList(nullList *obj); + typedef struct numberList numberList; struct numberList { @@ -162,7 +169,6 @@ void qapi_free_uint8List(uint8List *obj); #endif /* QAPI_TYPES_BUILTIN */ - typedef struct DummyForceArrays DummyForceArrays; typedef enum QapiErrorClass { @@ -218,4 +224,4 @@ struct X86CPUFeatureWordInfoList { void qapi_free_X86CPUFeatureWordInfoList(X86CPUFeatureWordInfoList *obj); -#endif +#endif /* QAPI_TYPES_H */ diff --git a/msvc/unicorn/qapi-visit.c b/msvc/unicorn/qapi-visit.c index b25d3714..3df9d4d7 100644 --- a/msvc/unicorn/qapi-visit.c +++ b/msvc/unicorn/qapi-visit.c @@ -1,16 +1,13 @@ /* AUTOMATICALLY GENERATED, DO NOT MODIFY */ /* - * schema-defined QAPI visitor functions + * Schema-defined QAPI visitors * * Copyright IBM, Corp. 2011 - * - * Authors: - * Anthony Liguori + * Copyright (C) 2014-2018 Red Hat, Inc. * * This work is licensed under the terms of the GNU LGPL, version 2.1 or later. * See the COPYING.LIB file in the top-level directory. - * */ #include "qemu/osdep.h" @@ -379,6 +376,37 @@ out: error_propagate(errp, err); } +void visit_type_nullList(Visitor *v, const char *name, nullList **obj, Error **errp) +{ + Error *err = NULL; + nullList *tail; + size_t size = sizeof(**obj); + + visit_start_list(v, name, (GenericList **)obj, size, &err); + if (err) { + goto out; + } + + for (tail = *obj; tail; + tail = (nullList *)visit_next_list(v, (GenericList *)tail, size)) { + visit_type_null(v, NULL, &tail->value, &err); + if (err) { + break; + } + } + + if (!err) { + visit_check_list(v, &err); + } + visit_end_list(v, (void **)obj); + if (err && visit_is_input(v)) { + qapi_free_nullList(*obj); + *obj = NULL; + } +out: + error_propagate(errp, err); +} + void visit_type_numberList(Visitor *v, const char *name, numberList **obj, Error **errp) { Error *err = NULL; diff --git a/msvc/unicorn/qapi-visit.h b/msvc/unicorn/qapi-visit.h index 865cd2a8..314d0631 100644 --- a/msvc/unicorn/qapi-visit.h +++ b/msvc/unicorn/qapi-visit.h @@ -1,16 +1,13 @@ /* AUTOMATICALLY GENERATED, DO NOT MODIFY */ /* - * schema-defined QAPI visitor functions + * Schema-defined QAPI visitors * * Copyright IBM, Corp. 2011 - * - * Authors: - * Anthony Liguori + * Copyright (C) 2014-2018 Red Hat, Inc. * * This work is licensed under the terms of the GNU LGPL, version 2.1 or later. * See the COPYING.LIB file in the top-level directory. - * */ #ifndef QAPI_VISIT_H @@ -32,6 +29,7 @@ void visit_type_int32List(Visitor *v, const char *name, int32List **obj, Error * void visit_type_int64List(Visitor *v, const char *name, int64List **obj, Error **errp); void visit_type_int8List(Visitor *v, const char *name, int8List **obj, Error **errp); void visit_type_intList(Visitor *v, const char *name, intList **obj, Error **errp); +void visit_type_nullList(Visitor *v, const char *name, nullList **obj, Error **errp); void visit_type_numberList(Visitor *v, const char *name, numberList **obj, Error **errp); void visit_type_sizeList(Visitor *v, const char *name, sizeList **obj, Error **errp); void visit_type_strList(Visitor *v, const char *name, strList **obj, Error **errp); @@ -42,7 +40,6 @@ void visit_type_uint8List(Visitor *v, const char *name, uint8List **obj, Error * #endif /* QAPI_VISIT_BUILTIN */ - void visit_type_DummyForceArrays_members(Visitor *v, DummyForceArrays *obj, Error **errp); void visit_type_DummyForceArrays(Visitor *v, const char *name, DummyForceArrays **obj, Error **errp); void visit_type_QapiErrorClass(Visitor *v, const char *name, QapiErrorClass *obj, Error **errp); @@ -52,4 +49,4 @@ void visit_type_X86CPUFeatureWordInfo(Visitor *v, const char *name, X86CPUFeatur void visit_type_X86CPUFeatureWordInfoList(Visitor *v, const char *name, X86CPUFeatureWordInfoList **obj, Error **errp); void visit_type_X86CPURegister32(Visitor *v, const char *name, X86CPURegister32 *obj, Error **errp); -#endif +#endif /* QAPI_VISIT_H */ diff --git a/qemu/Makefile b/qemu/Makefile index 2309e82d..2d21920f 100644 --- a/qemu/Makefile +++ b/qemu/Makefile @@ -26,7 +26,6 @@ CONFIG_USER_ONLY := $(if $(filter %-user,$(TARGET_DIRS)),y) CONFIG_ALL=y -include config-all-devices.mak -include $(SRC_PATH)/rules.mak config-host.mak: $(SRC_PATH)/configure @echo $@ is out-of-date, running configure @# TODO: The next lines include code which supports a smooth @@ -45,9 +44,11 @@ ifneq ($(filter-out %clean,$(MAKECMDGOALS)),$(if $(MAKECMDGOALS),,fail)) endif endif -GENERATED_HEADERS = config-host.h -GENERATED_HEADERS += qapi-types.h qapi-visit.h -GENERATED_SOURCES += qapi-types.c qapi-visit.c +include $(SRC_PATH)/rules.mak + +GENERATED_FILES = config-host.h +GENERATED_FILES += qapi-types.h qapi-visit.h +GENERATED_FILES += qapi-types.c qapi-visit.c # Don't try to regenerate Makefile or configure # We don't generate any of them @@ -141,22 +142,24 @@ util/module.o-cflags = -D'CONFIG_BLOCK_MODULES=$(block-modules)' ###################################################################### -gen-out-type = $(subst .,-,$(suffix $@)) - -qapi-py = $(SRC_PATH)/scripts/qapi.py $(SRC_PATH)/scripts/ordereddict.py +qapi-py = \ +$(SRC_PATH)/scripts/qapi/types.py \ +$(SRC_PATH)/scripts/qapi/visit.py \ +$(SRC_PATH)/scripts/qapi/common.py \ +$(SRC_PATH)/scripts/ordereddict.py \ +$(SRC_PATH)/scripts/qapi-gen.py qapi-modules = $(SRC_PATH)/qapi-schema.json $(SRC_PATH)/qapi/common.json -qapi-types.c qapi-types.h :\ -$(qapi-modules) $(SRC_PATH)/scripts/qapi-types.py $(qapi-py) - $(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi-types.py \ - $(gen-out-type) -o "." -b $<, \ - " GEN $@") -qapi-visit.c qapi-visit.h :\ -$(qapi-modules) $(SRC_PATH)/scripts/qapi-visit.py $(qapi-py) - $(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi-visit.py \ - $(gen-out-type) -o "." -b $<, \ - " GEN $@") +qapi-types.c qapi-types.h \ +qapi-visit.c qapi-visit.h \ +qapi-doc.texi: \ +qapi-gen-timestamp ; +qapi-gen-timestamp: $(qapi-modules) $(qapi-py) + $(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi-gen.py \ + -o "." -b $<, \ + "GEN","$(@:%-timestamp=%)") + @>$@ clean: # avoid old build problems by removing potentially incorrect old files @@ -164,9 +167,9 @@ clean: find . \( -name '*.l[oa]' -o -name '*.so' -o -name '*.dll' -o -name '*.mo' -o -name '*.[oda]' \) -type f -exec rm {} + rm -f $(filter-out %.tlb,$(TOOLS)) $(HELPERS-y) qemu-ga TAGS cscope.* *.pod *~ */*~ rm -rf .libs */.libs - @# May not be present in GENERATED_HEADERS - rm -f $(foreach f,$(GENERATED_HEADERS),$(f) $(f)-timestamp) - rm -f $(foreach f,$(GENERATED_SOURCES),$(f) $(f)-timestamp) + @# May not be present in GENERATED_FILES + rm -f $(foreach f,$(GENERATED_FILES),$(f) $(f)-timestamp) + rm -f qapi-gen-timestamp rm -rf qapi-generated for d in $(ALL_SUBDIRS); do \ if test -d $$d; then $(MAKE) -C $$d $@ || exit 1; fi; \ @@ -191,6 +194,6 @@ cscope: # Add a dependency on the generated files, so that they are always # rebuilt before other object files ifneq ($(filter-out %clean,$(MAKECMDGOALS)),$(if $(MAKECMDGOALS),,fail)) -Makefile: $(GENERATED_HEADERS) +Makefile: $(GENERATED_FILES) endif diff --git a/qemu/Makefile.target b/qemu/Makefile.target index 445d46f7..80139cee 100644 --- a/qemu/Makefile.target +++ b/qemu/Makefile.target @@ -123,6 +123,6 @@ ifneq ($(PROGS),) $(call install-prog,$(PROGS),$(DESTDIR)$(bindir)) endif -GENERATED_HEADERS += config-target.h -Makefile: $(GENERATED_HEADERS) +GENERATED_FILES += config-target.h +Makefile: $(GENERATED_FILES) diff --git a/qemu/scripts/qapi-gen.py b/qemu/scripts/qapi-gen.py new file mode 100644 index 00000000..24b4d8ff --- /dev/null +++ b/qemu/scripts/qapi-gen.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# QAPI generator +# +# This work is licensed under the terms of the GNU GPL, version 2 or later. +# See the COPYING file in the top-level directory. + +import sys +from qapi.common import parse_command_line, QAPISchema +from qapi.types import gen_types +from qapi.visit import gen_visit +# Unicorn: commented out +#from qapi.commands import gen_commands +#from qapi.events import gen_events +#from qapi.introspect import gen_introspect +#from qapi.doc import gen_doc + + +def main(argv): + (input_file, output_dir, prefix, opts) = \ + parse_command_line('bu', ['builtins', 'unmask-non-abi-names']) + + opt_builtins = False + opt_unmask = False + + for o, a in opts: + if o in ('-b', '--builtins'): + opt_builtins = True + if o in ('-u', '--unmask-non-abi-names'): + opt_unmask = True + + schema = QAPISchema(input_file) + + gen_types(schema, output_dir, prefix, opt_builtins) + gen_visit(schema, output_dir, prefix, opt_builtins) + # Unicorn: commented out + #gen_commands(schema, output_dir, prefix) + #gen_events(schema, output_dir, prefix) + #gen_introspect(schema, output_dir, prefix, opt_unmask) + #gen_doc(schema, output_dir, prefix) + + +if __name__ == '__main__': + main(sys.argv) diff --git a/qemu/scripts/qapi/__init__.py b/qemu/scripts/qapi/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qemu/scripts/qapi.py b/qemu/scripts/qapi/common.py similarity index 99% rename from qemu/scripts/qapi.py rename to qemu/scripts/qapi/common.py index d03848b8..a658f9d0 100644 --- a/qemu/scripts/qapi.py +++ b/qemu/scripts/qapi/common.py @@ -1937,17 +1937,15 @@ def parse_command_line(extra_options='', extra_long_options=[]): try: opts, args = getopt.gnu_getopt(sys.argv[1:], - 'chp:o:' + extra_options, - ['source', 'header', 'prefix=', - 'output-dir='] + extra_long_options) + 'p:o:' + extra_options, + ['prefix=', 'output-dir='] + + extra_long_options) except getopt.GetoptError as err: print("%s: %s" % (sys.argv[0], str(err)), file=sys.stderr) sys.exit(1) output_dir = '' prefix = '' - do_c = False - do_h = False extra_opts = [] for oa in opts: @@ -1961,23 +1959,15 @@ def parse_command_line(extra_options='', extra_long_options=[]): prefix = a elif o in ('-o', '--output-dir'): output_dir = a + '/' - elif o in ('-c', '--source'): - do_c = True - elif o in ('-h', '--header'): - do_h = True else: extra_opts.append(oa) - if not do_c and not do_h: - do_c = True - do_h = True - if len(args) != 1: print("%s: need exactly one argument" % sys.argv[0], file=sys.stderr) sys.exit(1) fname = args[0] - return (fname, output_dir, do_c, do_h, prefix, extra_opts) + return (fname, output_dir, prefix, extra_opts) # diff --git a/qemu/scripts/qapi-event.py b/qemu/scripts/qapi/events.py similarity index 93% rename from qemu/scripts/qapi-event.py rename to qemu/scripts/qapi/events.py index 6d81bc40..8791e604 100644 --- a/qemu/scripts/qapi-event.py +++ b/qemu/scripts/qapi/events.py @@ -12,7 +12,7 @@ This work is licensed under the terms of the GNU GPL, version 2. See the COPYING file in the top-level directory. """ -from qapi import * +from qapi.common import * def build_event_send_proto(name, arg_type, boxed): return 'void qapi_event_send_%(c_name)s(%(param)s)' % { @@ -170,11 +170,8 @@ class QAPISchemaGenEventVisitor(QAPISchemaVisitor): self.defn += gen_event_send(name, arg_type, boxed, self._enum_name) self._event_names.append(name) -def main(argv): - (input_file, output_dir, do_c, do_h, prefix, dummy) = parse_command_line() - +def gen_events(schema, output_dir, prefix): blurb = ' * Schema-defined QAPI/QMP events' - genc = QAPIGenC(blurb, __doc__) genh = QAPIGenH(blurb, __doc__) @@ -196,17 +193,9 @@ def main(argv): ''', prefix=prefix)) - schema = QAPISchema(input_file) vis = QAPISchemaGenEventVisitor(prefix) schema.visit(vis) genc.add(vis.defn) genh.add(vis.decl) - - if do_c: - genc.write(output_dir, prefix + 'qapi-event.c') - if do_h: - genh.write(output_dir, prefix + 'qapi-event.h') - - -if __name__ == '__main__': - main(sys.argv) + genc.write(output_dir, prefix + 'qapi-event.c') + genh.write(output_dir, prefix + 'qapi-event.h') diff --git a/qemu/scripts/qapi-types.py b/qemu/scripts/qapi/types.py similarity index 90% rename from qemu/scripts/qapi-types.py rename to qemu/scripts/qapi/types.py index f6e906a2..fee5e746 100644 --- a/qemu/scripts/qapi-types.py +++ b/qemu/scripts/qapi/types.py @@ -13,7 +13,7 @@ This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. """ -from qapi import * +from qapi.common import * # variants must be emitted before their container; track what has already @@ -240,22 +240,8 @@ class QAPISchemaGenTypeVisitor(QAPISchemaVisitor): self.decl += gen_object(name, None, [variants.tag_member], variants) self._gen_type_cleanup(name) -def main(argv): - # If you link code generated from multiple schemata, you want only one - # instance of the code for built-in types. Generate it only when - # opt_builtins, enabled by command line option -b. See also - # QAPISchemaGenTypeVisitor.visit_end(). - opt_builtins = False - - (input_file, output_dir, do_c, do_h, prefix, opts) = \ - parse_command_line('b', ['builtins']) - - for o, a in opts: - if o in ('-b', '--builtins'): - opt_builtins = True - +def gen_types(schema, output_dir, prefix, opt_builtins): blurb = ' * Schema-defined QAPI types' - genc = QAPIGenC(blurb, __doc__) genh = QAPIGenH(blurb, __doc__) @@ -271,17 +257,9 @@ def main(argv): /* #include "qapi/util.h" */ ''')) - schema = QAPISchema(input_file) vis = QAPISchemaGenTypeVisitor(opt_builtins) schema.visit(vis) genc.add(vis.defn) genh.add(vis.decl) - - if do_c: - genc.write(output_dir, prefix + 'qapi-types.c') - if do_h: - genh.write(output_dir, prefix + 'qapi-types.h') - - -if __name__ == '__main__': - main(sys.argv) + genc.write(output_dir, prefix + 'qapi-types.c') + genh.write(output_dir, prefix + 'qapi-types.h') diff --git a/qemu/scripts/qapi-visit.py b/qemu/scripts/qapi/visit.py similarity index 92% rename from qemu/scripts/qapi-visit.py rename to qemu/scripts/qapi/visit.py index 4103d8f9..85b58125 100644 --- a/qemu/scripts/qapi-visit.py +++ b/qemu/scripts/qapi/visit.py @@ -13,7 +13,7 @@ This work is licensed under the terms of the GNU GPL, version 2. See the COPYING file in the top-level directory. """ -from qapi import * +from qapi.common import * def gen_visit_decl(name, scalar=False): @@ -323,22 +323,8 @@ class QAPISchemaGenVisitVisitor(QAPISchemaVisitor): self.decl += gen_visit_decl(name) self.defn += gen_visit_alternate(name, variants) -def main(argv): - # If you link code generated from multiple schemata, you want only one - # instance of the code for built-in types. Generate it only when - # opt_builtins, enabled by command line option -b. See also - # QAPISchemaGenVisitVisitor.visit_end(). - opt_builtins = False - - (input_file, output_dir, do_c, do_h, prefix, opts) = \ - parse_command_line('b', ['builtins']) - - for o, a in opts: - if o in ('-b', '--builtins'): - opt_builtins = True - +def gen_visit(schema, output_dir, prefix, opt_builtins): blurb = ' * Schema-defined QAPI visitors' - genc = QAPIGenC(blurb, __doc__) genh = QAPIGenH(blurb, __doc__) @@ -358,17 +344,9 @@ def main(argv): ''', prefix=prefix)) - schema = QAPISchema(input_file) vis = QAPISchemaGenVisitVisitor(opt_builtins) schema.visit(vis) genc.add(vis.defn) genh.add(vis.decl) - - if do_c: - genc.write(output_dir, prefix + 'qapi-visit.c') - if do_h: - genh.write(output_dir, prefix + 'qapi-visit.h') - - -if __name__ == '__main__': - main(sys.argv) + genc.write(output_dir, prefix + 'qapi-visit.c') + genh.write(output_dir, prefix + 'qapi-visit.h')