qapi-gen: New common driver for code and doc generators

Whenever qapi-schema.json changes, we run six programs eleven times to
update eleven files. Similar for qga/qapi-schema.json. This is
silly. Replace the six programs by a single program that spits out
all eleven files.

The programs become modules in new Python package qapi, along with the
helper library. This requires moving them to scripts/qapi/. While
moving them, consistently drop executable mode bits.

Backports commit fb0bc835e56b894cbc7236294921e5393c786ad8 from qemu

.gitignore

@@ -32,6 +32,7 @@ qemu/qapi-types.h
 qemu/qapi-visit.h
 qemu/qapi-types.c
 qemu/qapi-visit.c
+qemu/qapi-gen-timestamp
 tags
 qemu/config-host.ld
 qemu/config.log

msvc/unicorn/qapi-types.c

@@ -1,17 +1,13 @@
 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
 
 /*
- * deallocation functions for schema-defined QAPI types
+ * Schema-defined QAPI types
  *
  * Copyright IBM, Corp. 2011
- *
+ * Copyright (c) 2013-2018 Red Hat Inc.
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *  Michael Roth      <mdroth@linux.vnet.ibm.com>
  *
  * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
  * See the COPYING.LIB file in the top-level directory.
- *
  */
 
 #include "qemu/osdep.h"
@@ -182,6 +178,19 @@ void qapi_free_intList(intList *obj)
     visit_free(v);
 }
 
+void qapi_free_nullList(nullList *obj)
+{
+    Visitor *v;
+
+    if (!obj) {
+        return;
+    }
+
+    v = qapi_dealloc_visitor_new();
+    visit_type_nullList(v, NULL, &obj, NULL);
+    visit_free(v);
+}
+
 void qapi_free_numberList(numberList *obj)
 {
     Visitor *v;

msvc/unicorn/qapi-types.h

@@ -1,21 +1,19 @@
 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
 
 /*
- * schema-defined QAPI types
+ * Schema-defined QAPI types
  *
  * Copyright IBM, Corp. 2011
- *
+ * Copyright (c) 2013-2018 Red Hat Inc.
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
  *
  * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
  * See the COPYING.LIB file in the top-level directory.
- *
  */
 
 #ifndef QAPI_TYPES_H
 #define QAPI_TYPES_H
 
+/* #include "qapi/util.h" */
 
 #ifndef QAPI_TYPES_BUILTIN
 #define QAPI_TYPES_BUILTIN
@@ -97,6 +95,15 @@ struct intList {
 
 void qapi_free_intList(intList *obj);
 
+typedef struct nullList nullList;
+
+struct nullList {
+    nullList *next;
+    QNull *value;
+};
+
+void qapi_free_nullList(nullList *obj);
+
 typedef struct numberList numberList;
 
 struct numberList {
@@ -162,7 +169,6 @@ void qapi_free_uint8List(uint8List *obj);
 
 #endif /* QAPI_TYPES_BUILTIN */
 
-
 typedef struct DummyForceArrays DummyForceArrays;
 
 typedef enum QapiErrorClass {
@@ -218,4 +224,4 @@ struct X86CPUFeatureWordInfoList {
 
 void qapi_free_X86CPUFeatureWordInfoList(X86CPUFeatureWordInfoList *obj);
 
-#endif
+#endif /* QAPI_TYPES_H */

msvc/unicorn/qapi-visit.c

@@ -1,16 +1,13 @@
 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
 
 /*
- * schema-defined QAPI visitor functions
+ * Schema-defined QAPI visitors
  *
  * Copyright IBM, Corp. 2011
- *
+ * Copyright (C) 2014-2018 Red Hat, Inc.
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
  *
  * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
  * See the COPYING.LIB file in the top-level directory.
- *
  */
 
 #include "qemu/osdep.h"
@@ -379,6 +376,37 @@ out:
     error_propagate(errp, err);
 }
 
+void visit_type_nullList(Visitor *v, const char *name, nullList **obj, Error **errp)
+{
+    Error *err = NULL;
+    nullList *tail;
+    size_t size = sizeof(**obj);
+
+    visit_start_list(v, name, (GenericList **)obj, size, &err);
+    if (err) {
+        goto out;
+    }
+
+    for (tail = *obj; tail;
+         tail = (nullList *)visit_next_list(v, (GenericList *)tail, size)) {
+        visit_type_null(v, NULL, &tail->value, &err);
+        if (err) {
+            break;
+        }
+    }
+
+    if (!err) {
+        visit_check_list(v, &err);
+    }
+    visit_end_list(v, (void **)obj);
+    if (err && visit_is_input(v)) {
+        qapi_free_nullList(*obj);
+        *obj = NULL;
+    }
+out:
+    error_propagate(errp, err);
+}
+
 void visit_type_numberList(Visitor *v, const char *name, numberList **obj, Error **errp)
 {
     Error *err = NULL;

msvc/unicorn/qapi-visit.h

@@ -1,16 +1,13 @@
 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
 
 /*
- * schema-defined QAPI visitor functions
+ * Schema-defined QAPI visitors
  *
  * Copyright IBM, Corp. 2011
- *
+ * Copyright (C) 2014-2018 Red Hat, Inc.
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
  *
  * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
  * See the COPYING.LIB file in the top-level directory.
- *
  */
 
 #ifndef QAPI_VISIT_H
@@ -32,6 +29,7 @@ void visit_type_int32List(Visitor *v, const char *name, int32List **obj, Error *
 void visit_type_int64List(Visitor *v, const char *name, int64List **obj, Error **errp);
 void visit_type_int8List(Visitor *v, const char *name, int8List **obj, Error **errp);
 void visit_type_intList(Visitor *v, const char *name, intList **obj, Error **errp);
+void visit_type_nullList(Visitor *v, const char *name, nullList **obj, Error **errp);
 void visit_type_numberList(Visitor *v, const char *name, numberList **obj, Error **errp);
 void visit_type_sizeList(Visitor *v, const char *name, sizeList **obj, Error **errp);
 void visit_type_strList(Visitor *v, const char *name, strList **obj, Error **errp);
@@ -42,7 +40,6 @@ void visit_type_uint8List(Visitor *v, const char *name, uint8List **obj, Error *
 
 #endif /* QAPI_VISIT_BUILTIN */
 
-
 void visit_type_DummyForceArrays_members(Visitor *v, DummyForceArrays *obj, Error **errp);
 void visit_type_DummyForceArrays(Visitor *v, const char *name, DummyForceArrays **obj, Error **errp);
 void visit_type_QapiErrorClass(Visitor *v, const char *name, QapiErrorClass *obj, Error **errp);
@@ -52,4 +49,4 @@ void visit_type_X86CPUFeatureWordInfo(Visitor *v, const char *name, X86CPUFeatur
 void visit_type_X86CPUFeatureWordInfoList(Visitor *v, const char *name, X86CPUFeatureWordInfoList **obj, Error **errp);
 void visit_type_X86CPURegister32(Visitor *v, const char *name, X86CPURegister32 *obj, Error **errp);
 
-#endif
+#endif /* QAPI_VISIT_H */

qemu/Makefile

@@ -26,7 +26,6 @@ CONFIG_USER_ONLY := $(if $(filter %-user,$(TARGET_DIRS)),y)
 CONFIG_ALL=y
 -include config-all-devices.mak
 
-include $(SRC_PATH)/rules.mak
 config-host.mak: $(SRC_PATH)/configure
 	@echo $@ is out-of-date, running configure
 	@# TODO: The next lines include code which supports a smooth
@@ -45,9 +44,11 @@ ifneq ($(filter-out %clean,$(MAKECMDGOALS)),$(if $(MAKECMDGOALS),,fail))
 endif
 endif
 
-GENERATED_HEADERS = config-host.h
+include $(SRC_PATH)/rules.mak
-GENERATED_HEADERS += qapi-types.h qapi-visit.h
+
-GENERATED_SOURCES += qapi-types.c qapi-visit.c
+GENERATED_FILES = config-host.h
+GENERATED_FILES += qapi-types.h qapi-visit.h
+GENERATED_FILES += qapi-types.c qapi-visit.c
 
 # Don't try to regenerate Makefile or configure
 # We don't generate any of them
@@ -141,22 +142,24 @@ util/module.o-cflags = -D'CONFIG_BLOCK_MODULES=$(block-modules)'
 
 ######################################################################
 
-gen-out-type = $(subst .,-,$(suffix $@))
+qapi-py = \
-
+$(SRC_PATH)/scripts/qapi/types.py \
-qapi-py = $(SRC_PATH)/scripts/qapi.py $(SRC_PATH)/scripts/ordereddict.py
+$(SRC_PATH)/scripts/qapi/visit.py \
+$(SRC_PATH)/scripts/qapi/common.py \
+$(SRC_PATH)/scripts/ordereddict.py \
+$(SRC_PATH)/scripts/qapi-gen.py
 
 qapi-modules = $(SRC_PATH)/qapi-schema.json $(SRC_PATH)/qapi/common.json
 
-qapi-types.c qapi-types.h :\
+qapi-types.c qapi-types.h \
-$(qapi-modules) $(SRC_PATH)/scripts/qapi-types.py $(qapi-py)
+qapi-visit.c qapi-visit.h \
-	$(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi-types.py \
+qapi-doc.texi: \
-		$(gen-out-type) -o "." -b $<, \
+qapi-gen-timestamp ;
-		"  GEN   $@")
+qapi-gen-timestamp: $(qapi-modules) $(qapi-py)
-qapi-visit.c qapi-visit.h :\
+	$(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi-gen.py \
-$(qapi-modules) $(SRC_PATH)/scripts/qapi-visit.py $(qapi-py)
+		-o "." -b $<, \
-	$(call quiet-command,$(PYTHON_UTF8) $(SRC_PATH)/scripts/qapi-visit.py \
+		"GEN","$(@:%-timestamp=%)")
-		$(gen-out-type) -o "." -b $<, \
+	@>$@
-		"  GEN   $@")
 
 clean:
 # avoid old build problems by removing potentially incorrect old files
@@ -164,9 +167,9 @@ clean:
 	find . \( -name '*.l[oa]' -o -name '*.so' -o -name '*.dll' -o -name '*.mo' -o -name '*.[oda]' \) -type f -exec rm {} +
 	rm -f $(filter-out %.tlb,$(TOOLS)) $(HELPERS-y) qemu-ga TAGS cscope.* *.pod *~ */*~
 	rm -rf .libs */.libs
-	@# May not be present in GENERATED_HEADERS
+	@# May not be present in GENERATED_FILES
-	rm -f $(foreach f,$(GENERATED_HEADERS),$(f) $(f)-timestamp)
+	rm -f $(foreach f,$(GENERATED_FILES),$(f) $(f)-timestamp)
-	rm -f $(foreach f,$(GENERATED_SOURCES),$(f) $(f)-timestamp)
+	rm -f qapi-gen-timestamp
 	rm -rf qapi-generated
 	for d in $(ALL_SUBDIRS); do \
 	if test -d $$d; then $(MAKE) -C $$d $@ || exit 1; fi; \
@@ -191,6 +194,6 @@ cscope:
 # Add a dependency on the generated files, so that they are always
 # rebuilt before other object files
 ifneq ($(filter-out %clean,$(MAKECMDGOALS)),$(if $(MAKECMDGOALS),,fail))
-Makefile: $(GENERATED_HEADERS)
+Makefile: $(GENERATED_FILES)
 endif
 

qemu/Makefile.target

@@ -123,6 +123,6 @@ ifneq ($(PROGS),)
 	$(call install-prog,$(PROGS),$(DESTDIR)$(bindir))
 endif
 
-GENERATED_HEADERS += config-target.h
+GENERATED_FILES += config-target.h
-Makefile: $(GENERATED_HEADERS)
+Makefile: $(GENERATED_FILES)
 

qemu/scripts/qapi-gen.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# QAPI generator
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or later.
+# See the COPYING file in the top-level directory.
+
+import sys
+from qapi.common import parse_command_line, QAPISchema
+from qapi.types import gen_types
+from qapi.visit import gen_visit
+# Unicorn: commented out
+#from qapi.commands import gen_commands
+#from qapi.events import gen_events
+#from qapi.introspect import gen_introspect
+#from qapi.doc import gen_doc
+
+
+def main(argv):
+    (input_file, output_dir, prefix, opts) = \
+        parse_command_line('bu', ['builtins', 'unmask-non-abi-names'])
+
+    opt_builtins = False
+    opt_unmask = False
+
+    for o, a in opts:
+        if o in ('-b', '--builtins'):
+            opt_builtins = True
+        if o in ('-u', '--unmask-non-abi-names'):
+            opt_unmask = True
+
+    schema = QAPISchema(input_file)
+
+    gen_types(schema, output_dir, prefix, opt_builtins)
+    gen_visit(schema, output_dir, prefix, opt_builtins)
+    # Unicorn: commented out
+    #gen_commands(schema, output_dir, prefix)
+    #gen_events(schema, output_dir, prefix)
+    #gen_introspect(schema, output_dir, prefix, opt_unmask)
+    #gen_doc(schema, output_dir, prefix)
+
+
+if __name__ == '__main__':
+    main(sys.argv)

qemu/scripts/qapi/common.py

@@ -1937,17 +1937,15 @@ def parse_command_line(extra_options='', extra_long_options=[]):
 
     try:
         opts, args = getopt.gnu_getopt(sys.argv[1:],
-                                       'chp:o:' + extra_options,
+                                       'p:o:' + extra_options,
-                                       ['source', 'header', 'prefix=',
+                                       ['prefix=', 'output-dir=']
-                                        'output-dir='] + extra_long_options)
+                                       + extra_long_options)
     except getopt.GetoptError as err:
         print("%s: %s" % (sys.argv[0], str(err)), file=sys.stderr)
         sys.exit(1)
 
     output_dir = ''
     prefix = ''
-    do_c = False
-    do_h = False
     extra_opts = []
 
     for oa in opts:
@@ -1961,23 +1959,15 @@ def parse_command_line(extra_options='', extra_long_options=[]):
             prefix = a
         elif o in ('-o', '--output-dir'):
             output_dir = a + '/'
-        elif o in ('-c', '--source'):
-            do_c = True
-        elif o in ('-h', '--header'):
-            do_h = True
         else:
             extra_opts.append(oa)
 
-    if not do_c and not do_h:
-        do_c = True
-        do_h = True
-
     if len(args) != 1:
         print("%s: need exactly one argument" % sys.argv[0], file=sys.stderr)
         sys.exit(1)
     fname = args[0]
 
-    return (fname, output_dir, do_c, do_h, prefix, extra_opts)
+    return (fname, output_dir, prefix, extra_opts)
 
 
 #

qemu/scripts/qapi/events.py

@@ -12,7 +12,7 @@ This work is licensed under the terms of the GNU GPL, version 2.
 See the COPYING file in the top-level directory.
 """
 
-from qapi import *
+from qapi.common import *
 
 def build_event_send_proto(name, arg_type, boxed):
      return 'void qapi_event_send_%(c_name)s(%(param)s)' % {
@@ -170,11 +170,8 @@ class QAPISchemaGenEventVisitor(QAPISchemaVisitor):
         self.defn += gen_event_send(name, arg_type, boxed, self._enum_name)
         self._event_names.append(name)
 
-def main(argv):
+def gen_events(schema, output_dir, prefix):
-    (input_file, output_dir, do_c, do_h, prefix, dummy) = parse_command_line()
-
     blurb = ' * Schema-defined QAPI/QMP events'
-
     genc = QAPIGenC(blurb, __doc__)
     genh = QAPIGenH(blurb, __doc__)
 
@@ -196,17 +193,9 @@ def main(argv):
 ''',
                    prefix=prefix))
 
-    schema = QAPISchema(input_file)
     vis = QAPISchemaGenEventVisitor(prefix)
     schema.visit(vis)
     genc.add(vis.defn)
     genh.add(vis.decl)
-
+    genc.write(output_dir, prefix + 'qapi-event.c')
-    if do_c:
+    genh.write(output_dir, prefix + 'qapi-event.h')
-        genc.write(output_dir, prefix + 'qapi-event.c')
-    if do_h:
-        genh.write(output_dir, prefix + 'qapi-event.h')
-
-
-if __name__ == '__main__':
-    main(sys.argv)

qemu/scripts/qapi/types.py

@@ -13,7 +13,7 @@ This work is licensed under the terms of the GNU GPL, version 2.
 # See the COPYING file in the top-level directory.
 """
 
-from qapi import *
+from qapi.common import *
 
 
 # variants must be emitted before their container; track what has already
@@ -240,22 +240,8 @@ class QAPISchemaGenTypeVisitor(QAPISchemaVisitor):
         self.decl += gen_object(name, None, [variants.tag_member], variants)
         self._gen_type_cleanup(name)
 
-def main(argv):
+def gen_types(schema, output_dir, prefix, opt_builtins):
-    # If you link code generated from multiple schemata, you want only one
-    # instance of the code for built-in types.  Generate it only when
-    # opt_builtins, enabled by command line option -b.  See also
-    # QAPISchemaGenTypeVisitor.visit_end().
-    opt_builtins = False
-
-    (input_file, output_dir, do_c, do_h, prefix, opts) = \
-        parse_command_line('b', ['builtins'])
-
-    for o, a in opts:
-        if o in ('-b', '--builtins'):
-            opt_builtins = True
-
     blurb = ' * Schema-defined QAPI types'
-
     genc = QAPIGenC(blurb, __doc__)
     genh = QAPIGenH(blurb, __doc__)
 
@@ -271,17 +257,9 @@ def main(argv):
 /* #include "qapi/util.h" */
 '''))
 
-    schema = QAPISchema(input_file)
     vis = QAPISchemaGenTypeVisitor(opt_builtins)
     schema.visit(vis)
     genc.add(vis.defn)
     genh.add(vis.decl)
-
+    genc.write(output_dir, prefix + 'qapi-types.c')
-    if do_c:
+    genh.write(output_dir, prefix + 'qapi-types.h')
-        genc.write(output_dir, prefix + 'qapi-types.c')
-    if do_h:
-        genh.write(output_dir, prefix + 'qapi-types.h')
-
-
-if __name__ == '__main__':
-    main(sys.argv)

qemu/scripts/qapi/visit.py

@@ -13,7 +13,7 @@ This work is licensed under the terms of the GNU GPL, version 2.
 See the COPYING file in the top-level directory.
 """
 
-from qapi import *
+from qapi.common import *
 
 
 def gen_visit_decl(name, scalar=False):
@@ -323,22 +323,8 @@ class QAPISchemaGenVisitVisitor(QAPISchemaVisitor):
         self.decl += gen_visit_decl(name)
         self.defn += gen_visit_alternate(name, variants)
 
-def main(argv):
+def gen_visit(schema, output_dir, prefix, opt_builtins):
-    # If you link code generated from multiple schemata, you want only one
-    # instance of the code for built-in types.  Generate it only when
-    # opt_builtins, enabled by command line option -b.  See also
-    # QAPISchemaGenVisitVisitor.visit_end().
-    opt_builtins = False
-
-    (input_file, output_dir, do_c, do_h, prefix, opts) = \
-        parse_command_line('b', ['builtins'])
-
-    for o, a in opts:
-        if o in ('-b', '--builtins'):
-            opt_builtins = True
-
     blurb = ' * Schema-defined QAPI visitors'
-
     genc = QAPIGenC(blurb, __doc__)
     genh = QAPIGenH(blurb, __doc__)
 
@@ -358,17 +344,9 @@ def main(argv):
 ''',
                    prefix=prefix))
 
-    schema = QAPISchema(input_file)
     vis = QAPISchemaGenVisitVisitor(opt_builtins)
     schema.visit(vis)
     genc.add(vis.defn)
     genh.add(vis.decl)
-
+    genc.write(output_dir, prefix + 'qapi-visit.c')
-    if do_c:
+    genh.write(output_dir, prefix + 'qapi-visit.h')
-        genc.write(output_dir, prefix + 'qapi-visit.c')
-    if do_h:
-        genh.write(output_dir, prefix + 'qapi-visit.h')
-
-
-if __name__ == '__main__':
-    main(sys.argv)