qiurui
/
Ubuntu-focal-kernel
mirror of https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/focal
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

UBUNTU: [Packaging] update annotations scripts 

BugLink: https://bugs.launchpad.net/bugs/1786013
Signed-off-by: Roxana Nicolescu <roxana.nicolescu@canonical.com>
This commit is contained in:
Roxana Nicolescu 2024-01-05 14:27:49 +01:00
parent 3df010e8d4
commit 8deb6abcf4
5 changed files with 625 additions and 390 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

288
debian/scripts/misc/annotations vendored
View File

@ -1,274 +1,34 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- mode: python -*- # -*- mode: python -*-
# Manage Ubuntu kernel .config and annotations
# Copyright © 2022 Canonical Ltd. # This file is not installed; it's just to run annotations from inside a source
# distribution without installing it in the system.
import sys import sys
# Prevent generating .pyc files on import
#
# We may end up adding these files to our git repos by mistake, so simply
# prevent generating them in advance.
#
# There's a tiny performance penalty with this, because python needs to
# re-generate the bytecode on-the-fly every time the script is executed, but
# this overhead is absolutely negligible compared the rest of the kernel build
# time.
sys.dont_write_bytecode = True sys.dont_write_bytecode = True
import os
import argparse
import json
from signal import signal, SIGPIPE, SIG_DFL
from kconfig.annotations import Annotation, KConfig import os # noqa: E402 Import not at top of file
from kconfig import run # noqa: E402 Import not at top of file
VERSION = '0.1'
SKIP_CONFIGS = (
# CONFIG_VERSION_SIGNATURE is dynamically set during the build
'CONFIG_VERSION_SIGNATURE',
# Allow to use a different versions of toolchain tools
'CONFIG_GCC_VERSION',
'CONFIG_CC_VERSION_TEXT',
'CONFIG_AS_VERSION',
'CONFIG_LD_VERSION',
'CONFIG_LLD_VERSION',
'CONFIG_CLANG_VERSION',
'CONFIG_PAHOLE_VERSION',
'CONFIG_RUSTC_VERSION_TEXT',
'CONFIG_BINDGEN_VERSION_TEXT',
)
def make_parser(): # Update PATH to make sure that annotations can be executed directly from the
parser = argparse.ArgumentParser( # source directory.
description='Manage Ubuntu kernel .config and annotations', def update_path():
) script_dir = os.path.dirname(os.path.abspath(__file__))
parser.add_argument('--version', '-v', action='version', version=f'%(prog)s {VERSION}') current_path = os.environ.get("PATH", "")
new_path = f"{script_dir}:{current_path}"
parser.add_argument('--file', '-f', action='store', os.environ["PATH"] = new_path
help='Pass annotations or .config file to be parsed')
parser.add_argument('--arch', '-a', action='store',
help='Select architecture')
parser.add_argument('--flavour', '-l', action='store',
help='Select flavour (default is "generic")')
parser.add_argument('--config', '-c', action='store',
help='Select a specific config option')
parser.add_argument('--query', '-q', action='store_true',
help='Query annotations')
parser.add_argument('--note', '-n', action='store',
help='Write a specific note to a config option in annotations')
parser.add_argument('--autocomplete', action='store_true',
help='Enable config bash autocomplete: `source <(annotations --autocomplete)`')
parser.add_argument('--source', '-t', action='store_true',
help='Jump to a config definition in the kernel source code')
ga = parser.add_argument_group(title='Action').add_mutually_exclusive_group(required=False)
ga.add_argument('--write', '-w', action='store',
metavar='VALUE', dest='value',
help='Set a specific config value in annotations (use \'null\' to remove)')
ga.add_argument('--export', '-e', action='store_true',
help='Convert annotations to .config format')
ga.add_argument('--import', '-i', action='store',
metavar="FILE", dest='import_file',
help='Import a full .config for a specific arch and flavour into annotations')
ga.add_argument('--update', '-u', action='store',
metavar="FILE", dest='update_file',
help='Import a partial .config into annotations (only resync configs specified in FILE)')
ga.add_argument('--check', '-k', action='store',
metavar="FILE", dest='check_file',
help='Validate kernel .config with annotations')
return parser
_ARGPARSER = make_parser() update_path()
exit(run.main())
def arg_fail(message):
print(message)
_ARGPARSER.print_usage()
sys.exit(1)
def print_result(config, res):
if res is not None and config not in res:
res = {config or '*': res}
print(json.dumps(res, indent=4))
def do_query(args):
if args.arch is None and args.flavour is not None:
arg_fail('error: --flavour requires --arch')
a = Annotation(args.file)
res = a.search_config(config=args.config, arch=args.arch, flavour=args.flavour)
print_result(args.config, res)
def do_autocomplete(args):
a = Annotation(args.file)
res = (c.removeprefix('CONFIG_') for c in a.search_config())
res_str = ' '.join(res)
print(f'complete -W "{res_str}" annotations')
def do_source(args):
if args.config is None:
arg_fail('error: --source requires --config')
if not os.path.exists('tags'):
print('tags not found in the current directory, try: `make tags`')
sys.exit(1)
os.system(f'vim -t {args.config}')
def do_note(args):
if args.config is None:
arg_fail('error: --note requires --config')
# Set the note in annotations
a = Annotation(args.file)
a.set(args.config, note=args.note)
# Save back to annotations
a.save(args.file)
# Query and print back the value
a = Annotation(args.file)
res = a.search_config(config=args.config)
print_result(args.config, res)
def do_write(args):
if args.config is None:
arg_fail('error: --write requires --config')
# Set the value in annotations ('null' means remove)
a = Annotation(args.file)
if args.value == 'null':
a.remove(args.config, arch=args.arch, flavour=args.flavour)
else:
a.set(args.config, arch=args.arch, flavour=args.flavour, value=args.value, note=args.note)
# Save back to annotations
a.save(args.file)
# Query and print back the value
a = Annotation(args.file)
res = a.search_config(config=args.config)
print_result(args.config, res)
def do_export(args):
if args.arch is None:
arg_fail('error: --export requires --arch')
a = Annotation(args.file)
conf = a.search_config(config=args.config, arch=args.arch, flavour=args.flavour)
if conf:
print(a.to_config(conf))
def do_import(args):
if args.arch is None:
arg_fail('error: --arch is required with --import')
if args.flavour is None:
arg_fail('error: --flavour is required with --import')
if args.config is not None:
arg_fail('error: --config cannot be used with --import (try --update)')
# Merge with the current annotations
a = Annotation(args.file)
c = KConfig(args.import_file)
a.update(c, arch=args.arch, flavour=args.flavour)
# Save back to annotations
a.save(args.file)
def do_update(args):
if args.arch is None:
arg_fail('error: --arch is required with --update')
# Merge with the current annotations
a = Annotation(args.file)
c = KConfig(args.update_file)
if args.config is None:
configs = list(set(c.config.keys()) - set(SKIP_CONFIGS))
if configs:
a.update(c, arch=args.arch, flavour=args.flavour, configs=configs)
# Save back to annotations
a.save(args.file)
def do_check(args):
# Determine arch and flavour
if args.arch is None:
arg_fail('error: --arch is required with --check')
print(f"check-config: loading annotations from {args.file}")
total = good = ret = 0
# Load annotations settings
a = Annotation(args.file)
a_configs = a.search_config(arch=args.arch, flavour=args.flavour).keys()
# Parse target .config
c = KConfig(args.check_file)
c_configs = c.config.keys()
# Validate .config against annotations
for conf in sorted(a_configs | c_configs):
if conf in SKIP_CONFIGS:
continue
entry = a.search_config(config=conf, arch=args.arch, flavour=args.flavour)
expected = entry[conf] if entry else '-'
value = c.config[conf] if conf in c.config else '-'
if value != expected:
policy = a.config[conf] if conf in a.config else 'undefined'
if 'policy' in policy:
policy = f"policy<{policy['policy']}>"
print(f"check-config: FAIL: ({value} != {expected}): {conf} {policy})")
ret = 1
else:
good += 1
total += 1
print(f"check-config: {good}/{total} checks passed -- exit {ret}")
sys.exit(ret)
def autodetect_annotations(args):
if args.file:
return
# If --file/-f isn't specified try to automatically determine the right
# location of the annotations file looking at debian/debian.env.
try:
with open('debian/debian.env', 'rt', encoding='utf-8') as fd:
args.file = fd.read().rstrip().split('=')[1] + '/config/annotations'
except (FileNotFoundError, IndexError):
arg_fail('error: could not determine DEBDIR, try using: --file/-f')
def main():
# Prevent broken pipe errors when showing output in pipe to other tools
# (less for example)
signal(SIGPIPE, SIG_DFL)
# Main annotations program
args = _ARGPARSER.parse_args()
autodetect_annotations(args)
if args.config and not args.config.startswith('CONFIG_'):
args.config = 'CONFIG_' + args.config
if args.value:
do_write(args)
elif args.note:
do_note(args)
elif args.export:
do_export(args)
elif args.import_file:
do_import(args)
elif args.update_file:
do_update(args)
elif args.check_file:
do_check(args)
elif args.autocomplete:
do_autocomplete(args)
elif args.source:
do_source(args)
else:
do_query(args)
if __name__ == '__main__':
main()

332
debian/scripts/misc/kconfig/annotations.py vendored
View File

@ -1,4 +1,3 @@
#!/usr/bin/env python
# -*- mode: python -*- # -*- mode: python -*-
# python module to manage Ubuntu kernel .config and annotations # python module to manage Ubuntu kernel .config and annotations
# Copyright © 2022 Canonical Ltd. # Copyright © 2022 Canonical Ltd.
@ -12,26 +11,29 @@ from abc import abstractmethod
from ast import literal_eval from ast import literal_eval
from os.path import dirname, abspath from os.path import dirname, abspath
from kconfig.version import ANNOTATIONS_FORMAT_VERSION
class Config():
def __init__(self, fname): class Config:
def __init__(self, fname, do_include=True):
""" """
Basic configuration file object Basic configuration file object
""" """
self.fname = fname self.fname = fname
self.config = {} self.config = {}
self.do_include = do_include
raw_data = self._load(fname) raw_data = self._load(fname)
self._parse(raw_data) self._parse(raw_data)
@staticmethod @staticmethod
def _load(fname: str) -> str: def _load(fname: str) -> str:
with open(fname, 'rt', encoding='utf-8') as fd: with open(fname, "rt", encoding="utf-8") as fd:
data = fd.read() data = fd.read()
return data.rstrip() return data.rstrip()
def __str__(self): def __str__(self):
""" Return a JSON representation of the config """ """Return a JSON representation of the config"""
return json.dumps(self.config, indent=4) return json.dumps(self.config, indent=4)
@abstractmethod @abstractmethod
@ -44,14 +46,15 @@ class KConfig(Config):
Parse a .config file, individual config options can be accessed via Parse a .config file, individual config options can be accessed via
.config[<CONFIG_OPTION>] .config[<CONFIG_OPTION>]
""" """
def _parse(self, data: str): def _parse(self, data: str):
self.config = {} self.config = {}
for line in data.splitlines(): for line in data.splitlines():
m = re.match(r'^# (CONFIG_.*) is not set$', line) m = re.match(r"^# (CONFIG_.*) is not set$", line)
if m: if m:
self.config[m.group(1)] = literal_eval("'n'") self.config[m.group(1)] = literal_eval("'n'")
continue continue
m = re.match(r'^(CONFIG_[A-Za-z0-9_]+)=(.*)$', line) m = re.match(r"^(CONFIG_[A-Za-z0-9_]+)=(.*)$", line)
if m: if m:
self.config[m.group(1)] = literal_eval("'" + m.group(2) + "'") self.config[m.group(1)] = literal_eval("'" + m.group(2) + "'")
continue continue
@ -61,12 +64,13 @@ class Annotation(Config):
""" """
Parse body of annotations file Parse body of annotations file
""" """
def _parse_body(self, data: str, parent=True): def _parse_body(self, data: str, parent=True):
for line in data.splitlines(): for line in data.splitlines():
# Replace tabs with spaces, squeeze multiple into singles and # Replace tabs with spaces, squeeze multiple into singles and
# remove leading and trailing spaces # remove leading and trailing spaces
line = line.replace('\t', ' ') line = line.replace("\t", " ")
line = re.sub(r' +', ' ', line) line = re.sub(r" +", " ", line)
line = line.strip() line = line.strip()
# Ignore empty lines # Ignore empty lines
@ -74,12 +78,12 @@ class Annotation(Config):
continue continue
# Catpure flavors of included files # Catpure flavors of included files
if line.startswith('# FLAVOUR: '): if line.startswith("# FLAVOUR: "):
self.include_flavour += line.split(' ')[2:] self.include_flavour += line.split(" ")[2:]
continue continue
# Ignore comments # Ignore comments
if line.startswith('#'): if line.startswith("#"):
continue continue
# Handle includes (recursively) # Handle includes (recursively)
@ -87,46 +91,59 @@ class Annotation(Config):
if m: if m:
if parent: if parent:
self.include.append(m.group(1)) self.include.append(m.group(1))
include_fname = dirname(abspath(self.fname)) + '/' + m.group(1) if self.do_include:
include_data = self._load(include_fname) include_fname = dirname(abspath(self.fname)) + "/" + m.group(1)
self._parse_body(include_data, parent=False) include_data = self._load(include_fname)
self._parse_body(include_data, parent=False)
continue continue
# Handle policy and note lines # Handle policy and note lines
if re.match(r'.* (policy|note)<', line): if re.match(r".* (policy|note)<", line):
try: try:
conf = line.split(' ')[0] conf = line.split(" ")[0]
if conf in self.config: if conf in self.config:
entry = self.config[conf] entry = self.config[conf]
else: else:
entry = {'policy': {}} entry = {"policy": {}}
match = False match = False
m = re.match(r'.* policy<(.*?)>', line) m = re.match(r".* policy<(.*?)>", line)
if m: if m:
match = True match = True
try: # Update the previous entry considering potential overrides:
entry['policy'] |= literal_eval(m.group(1)) # - if the new entry is adding a rule for a new
except TypeError: # arch/flavour, simply add that
entry['policy'] = {**entry['policy'], **literal_eval(m.group(1))} # - if the new entry is overriding a previous
# arch-flavour item, then overwrite that item
# - if the new entry is overriding a whole arch, then
# remove all the previous flavour rules of that arch
new_entry = literal_eval(m.group(1))
for key in new_entry:
if key in self.arch:
for flavour_key in list(entry["policy"].keys()):
if flavour_key.startswith(key):
del entry["policy"][flavour_key]
entry["policy"][key] = new_entry[key]
else:
entry["policy"][key] = new_entry[key]
m = re.match(r'.* note<(.*?)>', line) m = re.match(r".* note<(.*?)>", line)
if m: if m:
entry['oneline'] = match entry["oneline"] = match
match = True match = True
entry['note'] = "'" + m.group(1).replace("'", '') + "'" entry["note"] = "'" + m.group(1).replace("'", "") + "'"
if not match: if not match:
raise SyntaxError('syntax error') raise SyntaxError("syntax error")
self.config[conf] = entry self.config[conf] = entry
except Exception as e: except Exception as e:
raise SyntaxError(str(e) + f', line = {line}') from e raise SyntaxError(str(e) + f", line = {line}") from e
continue continue
# Invalid line # Invalid line
raise SyntaxError(f'invalid line: {line}') raise SyntaxError(f"invalid line: {line}")
def _parse(self, data: str): def _legacy_parse(self, data: str):
""" """
Parse main annotations file, individual config options can be accessed Parse main annotations file, individual config options can be accessed
via self.config[<CONFIG_OPTION>] via self.config[<CONFIG_OPTION>]
@ -136,35 +153,86 @@ class Annotation(Config):
self.flavour = [] self.flavour = []
self.flavour_dep = {} self.flavour_dep = {}
self.include = [] self.include = []
self.header = '' self.header = ""
self.include_flavour = [] self.include_flavour = []
# Parse header (only main header will considered, headers in includes # Parse header (only main header will considered, headers in includes
# will be treated as comments) # will be treated as comments)
for line in data.splitlines(): for line in data.splitlines():
if re.match(r'^#.*', line): if re.match(r"^#.*", line):
m = re.match(r'^# ARCH: (.*)', line) m = re.match(r"^# ARCH: (.*)", line)
if m: if m:
self.arch = list(m.group(1).split(' ')) self.arch = list(m.group(1).split(" "))
m = re.match(r'^# FLAVOUR: (.*)', line) m = re.match(r"^# FLAVOUR: (.*)", line)
if m: if m:
self.flavour = list(m.group(1).split(' ')) self.flavour = list(m.group(1).split(" "))
m = re.match(r'^# FLAVOUR_DEP: (.*)', line) m = re.match(r"^# FLAVOUR_DEP: (.*)", line)
if m: if m:
self.flavour_dep = literal_eval(m.group(1)) self.flavour_dep = literal_eval(m.group(1))
self.header += line + "\n" self.header += line + "\n"
else: else:
break break
# Parse body (handle includes recursively) # Return an error if architectures are not defined
if not self.arch:
raise SyntaxError("ARCH not defined in annotations")
# Return an error if flavours are not defined
if not self.flavour:
raise SyntaxError("FLAVOUR not defined in annotations")
# Parse body
self._parse_body(data) self._parse_body(data)
# Sanity check: Verify that all FLAVOUR_DEP flavors are valid # Sanity check: Verify that all FLAVOUR_DEP flavors are valid
for src, tgt in self.flavour_dep.items(): if self.do_include:
if src not in self.flavour: for src, tgt in self.flavour_dep.items():
raise SyntaxError(f'Invalid source flavour in FLAVOUR_DEP: {src}') if src not in self.flavour:
if tgt not in self.include_flavour: raise SyntaxError(f"Invalid source flavour in FLAVOUR_DEP: {src}")
raise SyntaxError(f'Invalid target flavour in FLAVOUR_DEP: {tgt}') if tgt not in self.include_flavour:
raise SyntaxError(f"Invalid target flavour in FLAVOUR_DEP: {tgt}")
def _json_parse(self, data, is_included=False):
data = json.loads(data)
# Check if version is supported
version = data["attributes"]["_version"]
if version > ANNOTATIONS_FORMAT_VERSION:
raise SyntaxError(f"annotations format version {version} not supported")
# Check for top-level annotations vs imported annotations
if not is_included:
self.config = data["config"]
self.arch = data["attributes"]["arch"]
self.flavour = data["attributes"]["flavour"]
self.flavour_dep = data["attributes"]["flavour_dep"]
self.include = data["attributes"]["include"]
self.include_flavour = []
else:
# We are procesing an imported annotations, so merge all the
# configs and attributes.
try:
self.config = data["config"] | self.config
except TypeError:
self.config = {**self.config, **data["config"]}
self.arch = list(set(self.arch) | set(data["attributes"]["arch"]))
self.flavour = list(set(self.flavour) | set(data["attributes"]["flavour"]))
self.include_flavour = list(set(self.include_flavour) | set(data["attributes"]["flavour"]))
self.flavour_dep = self.flavour_dep | data["attributes"]["flavour_dep"]
# Handle recursive inclusions
if self.do_include:
for f in data["attributes"]["include"]:
include_fname = dirname(abspath(self.fname)) + "/" + f
data = self._load(include_fname)
self._json_parse(data, is_included=True)
def _parse(self, data: str):
# Try to parse the legacy format first, otherwise use the new JSON
# format.
try:
self._legacy_parse(data)
except SyntaxError:
self._json_parse(data, is_included=False)
def _remove_entry(self, config: str): def _remove_entry(self, config: str):
if self.config[config]: if self.config[config]:
@ -175,34 +243,40 @@ class Annotation(Config):
return return
if arch is not None: if arch is not None:
if flavour is not None: if flavour is not None:
flavour = f'{arch}-{flavour}' flavour = f"{arch}-{flavour}"
else: else:
flavour = arch flavour = arch
del self.config[config]['policy'][flavour] del self.config[config]["policy"][flavour]
if not self.config[config]['policy']: if not self.config[config]["policy"]:
self._remove_entry(config) self._remove_entry(config)
else: else:
self._remove_entry(config) self._remove_entry(config)
def set(self, config: str, arch: str = None, flavour: str = None, def set(
value: str = None, note: str = None): self,
config: str,
arch: str = None,
flavour: str = None,
value: str = None,
note: str = None,
):
if value is not None: if value is not None:
if config not in self.config: if config not in self.config:
self.config[config] = {'policy': {}} self.config[config] = {"policy": {}}
if arch is not None: if arch is not None:
if flavour is not None: if flavour is not None:
flavour = f'{arch}-{flavour}' flavour = f"{arch}-{flavour}"
else: else:
flavour = arch flavour = arch
self.config[config]['policy'][flavour] = value self.config[config]["policy"][flavour] = value
else: else:
for a in self.arch: for a in self.arch:
self.config[config]['policy'][a] = value self.config[config]["policy"][a] = value
if note is not None: if note is not None:
self.config[config]['note'] = "'" + note.replace("'", '') + "'" self.config[config]["note"] = "'" + note.replace("'", "") + "'"
def update(self, c: KConfig, arch: str, flavour: str = None, configs: list = None): def update(self, c: KConfig, arch: str, flavour: str = None, configs: list = None):
""" Merge configs from a Kconfig object into Annotation object """ """Merge configs from a Kconfig object into Annotation object"""
# Determine if we need to import all configs or a single config # Determine if we need to import all configs or a single config
if not configs: if not configs:
@ -210,72 +284,75 @@ class Annotation(Config):
try: try:
configs |= self.search_config(arch=arch, flavour=flavour).keys() configs |= self.search_config(arch=arch, flavour=flavour).keys()
except TypeError: except TypeError:
configs = {**configs, **self.search_config(arch=arch, flavour=flavour).keys()} configs = {
**configs,
**self.search_config(arch=arch, flavour=flavour).keys(),
}
# Import configs from the Kconfig object into Annotations # Import configs from the Kconfig object into Annotations
flavour_arg = flavour flavour_arg = flavour
if flavour is not None: if flavour is not None:
flavour = arch + f'-{flavour}' flavour = arch + f"-{flavour}"
else: else:
flavour = arch flavour = arch
for conf in configs: for conf in configs:
if conf in c.config: if conf in c.config:
val = c.config[conf] val = c.config[conf]
else: else:
val = '-' val = "-"
if conf in self.config: if conf in self.config:
if 'policy' in self.config[conf]: if "policy" in self.config[conf]:
# Add a TODO if a config with a note is changing and print # Add a TODO if a config with a note is changing and print
# a warning # a warning
old_val = self.search_config(config=conf, arch=arch, flavour=flavour_arg) old_val = self.search_config(config=conf, arch=arch, flavour=flavour_arg)
if old_val: if old_val:
old_val = old_val[conf] old_val = old_val[conf]
if val != old_val and "note" in self.config[conf]: if val != old_val and "note" in self.config[conf]:
self.config[conf]['note'] = "TODO: update note" self.config[conf]["note"] = "TODO: update note"
print(f"WARNING: {conf} changed from {old_val} to {val}, updating note") print(f"WARNING: {conf} changed from {old_val} to {val}, updating note")
self.config[conf]['policy'][flavour] = val self.config[conf]["policy"][flavour] = val
else: else:
self.config[conf]['policy'] = {flavour: val} self.config[conf]["policy"] = {flavour: val}
else: else:
self.config[conf] = {'policy': {flavour: val}} self.config[conf] = {"policy": {flavour: val}}
def _compact(self): def _compact(self):
# Try to remove redundant settings: if the config value of a flavour is # Try to remove redundant settings: if the config value of a flavour is
# the same as the one of the main arch simply drop it. # the same as the one of the main arch simply drop it.
for conf in self.config.copy(): for conf in self.config.copy():
if 'policy' not in self.config[conf]: if "policy" not in self.config[conf]:
continue continue
for flavour in self.flavour: for flavour in self.flavour:
if flavour not in self.config[conf]['policy']: if flavour not in self.config[conf]["policy"]:
continue continue
m = re.match(r'^(.*?)-(.*)$', flavour) m = re.match(r"^(.*?)-(.*)$", flavour)
if not m: if not m:
continue continue
arch = m.group(1) arch = m.group(1)
if arch in self.config[conf]['policy']: if arch in self.config[conf]["policy"]:
if self.config[conf]['policy'][flavour] == self.config[conf]['policy'][arch]: if self.config[conf]["policy"][flavour] == self.config[conf]["policy"][arch]:
del self.config[conf]['policy'][flavour] del self.config[conf]["policy"][flavour]
continue continue
if flavour not in self.flavour_dep: if flavour not in self.flavour_dep:
continue continue
generic = self.flavour_dep[flavour] generic = self.flavour_dep[flavour]
if generic in self.config[conf]['policy']: if generic in self.config[conf]["policy"]:
if self.config[conf]['policy'][flavour] == self.config[conf]['policy'][generic]: if self.config[conf]["policy"][flavour] == self.config[conf]["policy"][generic]:
del self.config[conf]['policy'][flavour] del self.config[conf]["policy"][flavour]
continue continue
# Remove rules for flavours / arches that are not supported (not # Remove rules for flavours / arches that are not supported (not
# listed in the annotations header). # listed in the annotations header).
for flavour in self.config[conf]['policy'].copy(): for flavour in self.config[conf]["policy"].copy():
if flavour not in list(set(self.arch + self.flavour)): if flavour not in list(set(self.arch + self.flavour)):
del self.config[conf]['policy'][flavour] del self.config[conf]["policy"][flavour]
# Remove configs that are all undefined across all arches/flavours # Remove configs that are all undefined across all arches/flavours
# (unless we have includes) # (unless we have includes)
if not self.include: if not self.include:
if 'policy' in self.config[conf]: if "policy" in self.config[conf]:
if list(set(self.config[conf]['policy'].values())) == ['-']: if list(set(self.config[conf]["policy"].values())) == ["-"]:
self.config[conf]['policy'] = {} self.config[conf]["policy"] = {}
# Drop empty rules # Drop empty rules
if not self.config[conf]['policy']: if not self.config[conf]["policy"]:
del self.config[conf] del self.config[conf]
else: else:
# Compact same value across all flavour within the same arch # Compact same value across all flavour within the same arch
@ -283,16 +360,16 @@ class Annotation(Config):
arch_flavours = [i for i in self.flavour if i.startswith(arch)] arch_flavours = [i for i in self.flavour if i.startswith(arch)]
value = None value = None
for flavour in arch_flavours: for flavour in arch_flavours:
if flavour not in self.config[conf]['policy']: if flavour not in self.config[conf]["policy"]:
break break
if value is None: if value is None:
value = self.config[conf]['policy'][flavour] value = self.config[conf]["policy"][flavour]
elif value != self.config[conf]['policy'][flavour]: elif value != self.config[conf]["policy"][flavour]:
break break
else: else:
for flavour in arch_flavours: for flavour in arch_flavours:
del self.config[conf]['policy'][flavour] del self.config[conf]["policy"][flavour]
self.config[conf]['policy'][arch] = value self.config[conf]["policy"][arch] = value
# After the first round of compaction we may end up having configs that # After the first round of compaction we may end up having configs that
# are undefined across all arches, so do another round of compaction to # are undefined across all arches, so do another round of compaction to
# drop these settings that are not needed anymore # drop these settings that are not needed anymore
@ -300,34 +377,34 @@ class Annotation(Config):
if not self.include: if not self.include:
for conf in self.config.copy(): for conf in self.config.copy():
# Remove configs that are all undefined across all arches/flavours # Remove configs that are all undefined across all arches/flavours
if 'policy' in self.config[conf]: if "policy" in self.config[conf]:
if list(set(self.config[conf]['policy'].values())) == ['-']: if list(set(self.config[conf]["policy"].values())) == ["-"]:
self.config[conf]['policy'] = {} self.config[conf]["policy"] = {}
# Drop empty rules # Drop empty rules
if not self.config[conf]['policy']: if not self.config[conf]["policy"]:
del self.config[conf] del self.config[conf]
@staticmethod @staticmethod
def _sorted(config): def _sorted(config):
""" Sort configs alphabetically but return configs with a note first """ """Sort configs alphabetically but return configs with a note first"""
w_note = [] w_note = []
wo_note = [] wo_note = []
for c in sorted(config): for c in sorted(config):
if 'note' in config[c]: if "note" in config[c]:
w_note.append(c) w_note.append(c)
else: else:
wo_note.append(c) wo_note.append(c)
return w_note + wo_note return w_note + wo_note
def save(self, fname: str): def save(self, fname: str):
""" Save annotations data to the annotation file """ """Save annotations data to the annotation file"""
# Compact annotations structure # Compact annotations structure
self._compact() self._compact()
# Save annotations to disk # Save annotations to disk
with tempfile.NamedTemporaryFile(mode='w+t', delete=False) as tmp: with tempfile.NamedTemporaryFile(mode="w+t", delete=False) as tmp:
# Write header # Write header
tmp.write(self.header + '\n') tmp.write(self.header + "\n")
# Write includes # Write includes
for i in self.include: for i in self.include:
@ -344,40 +421,43 @@ class Annotation(Config):
marker = False marker = False
for conf in self._sorted(self.config): for conf in self._sorted(self.config):
new_val = self.config[conf] new_val = self.config[conf]
if 'policy' not in new_val: if "policy" not in new_val:
continue continue
# If new_val is a subset of old_val, skip it unless there are # If new_val is a subset of old_val, skip it unless there are
# new notes that are different than the old ones. # new notes that are different than the old ones.
old_val = tmp_a.config.get(conf) old_val = tmp_a.config.get(conf)
if old_val and 'policy' in old_val: if old_val and "policy" in old_val:
try: try:
can_skip = old_val['policy'] == old_val['policy'] | new_val['policy'] can_skip = old_val["policy"] == old_val["policy"] | new_val["policy"]
except TypeError: except TypeError:
can_skip = old_val['policy'] == {**old_val['policy'], **new_val['policy']} can_skip = old_val["policy"] == {
**old_val["policy"],
**new_val["policy"],
}
if can_skip: if can_skip:
if 'note' not in new_val: if "note" not in new_val:
continue continue
if 'note' in old_val and 'note' in new_val: if "note" in old_val and "note" in new_val:
if old_val['note'] == new_val['note']: if old_val["note"] == new_val["note"]:
continue continue
# Write out the policy (and note) line(s) # Write out the policy (and note) line(s)
val = dict(sorted(new_val['policy'].items())) val = dict(sorted(new_val["policy"].items()))
line = f"{conf : <47} policy<{val}>" line = f"{conf : <47} policy<{val}>"
if 'note' in new_val: if "note" in new_val:
val = new_val['note'] val = new_val["note"]
if new_val.get('oneline', False): if new_val.get("oneline", False):
# Single line # Single line
line += f' note<{val}>' line += f" note<{val}>"
else: else:
# Separate policy and note lines, # Separate policy and note lines,
# followed by an empty line # followed by an empty line
line += f'\n{conf : <47} note<{val}>\n' line += f"\n{conf : <47} note<{val}>\n"
elif not marker: elif not marker:
# Write out a marker indicating the start of annotations # Write out a marker indicating the start of annotations
# without notes # without notes
tmp.write('\n# ---- Annotations without notes ----\n\n') tmp.write("\n# ---- Annotations without notes ----\n\n")
marker = True marker = True
tmp.write(line + "\n") tmp.write(line + "\n")
@ -386,10 +466,10 @@ class Annotation(Config):
shutil.move(tmp.name, fname) shutil.move(tmp.name, fname)
def search_config(self, config: str = None, arch: str = None, flavour: str = None) -> dict: def search_config(self, config: str = None, arch: str = None, flavour: str = None) -> dict:
""" Return config value of a specific config option or architecture """ """Return config value of a specific config option or architecture"""
if flavour is None: if flavour is None:
flavour = 'generic' flavour = "generic"
flavour = f'{arch}-{flavour}' flavour = f"{arch}-{flavour}"
if flavour in self.flavour_dep: if flavour in self.flavour_dep:
generic = self.flavour_dep[flavour] generic = self.flavour_dep[flavour]
else: else:
@ -401,14 +481,14 @@ class Annotation(Config):
# Get config options of a specific architecture # Get config options of a specific architecture
ret = {} ret = {}
for c, val in self.config.items(): for c, val in self.config.items():
if 'policy' not in val: if "policy" not in val:
continue continue
if flavour in val['policy']: if flavour in val["policy"]:
ret[c] = val['policy'][flavour] ret[c] = val["policy"][flavour]
elif generic != flavour and generic in val['policy']: elif generic != flavour and generic in val["policy"]:
ret[c] = val['policy'][generic] ret[c] = val["policy"][generic]
elif arch in val['policy']: elif arch in val["policy"]:
ret[c] = val['policy'][arch] ret[c] = val["policy"][arch]
return ret return ret
if config is not None and arch is None: if config is not None and arch is None:
# Get a specific config option for all architectures # Get a specific config option for all architectures
@ -416,24 +496,24 @@ class Annotation(Config):
if config is not None and arch is not None: if config is not None and arch is not None:
# Get a specific config option for a specific architecture # Get a specific config option for a specific architecture
if config in self.config: if config in self.config:
if 'policy' in self.config[config]: if "policy" in self.config[config]:
if flavour in self.config[config]['policy']: if flavour in self.config[config]["policy"]:
return {config: self.config[config]['policy'][flavour]} return {config: self.config[config]["policy"][flavour]}
if generic != flavour and generic in self.config[config]['policy']: if generic != flavour and generic in self.config[config]["policy"]:
return {config: self.config[config]['policy'][generic]} return {config: self.config[config]["policy"][generic]}
if arch in self.config[config]['policy']: if arch in self.config[config]["policy"]:
return {config: self.config[config]['policy'][arch]} return {config: self.config[config]["policy"][arch]}
return None return None
@staticmethod @staticmethod
def to_config(data: dict) -> str: def to_config(data: dict) -> str:
""" Convert annotations data to .config format """ """Convert annotations data to .config format"""
s = '' s = ""
for c in data: for c in data:
v = data[c] v = data[c]
if v == 'n': if v == "n":
s += f"# {c} is not set\n" s += f"# {c} is not set\n"
elif v == '-': elif v == "-":
pass pass
else: else:
s += f"{c}={v}\n" s += f"{c}={v}\n"

365
debian/scripts/misc/kconfig/run.py vendored Normal file
View File

@ -0,0 +1,365 @@
# -*- mode: python -*-
# Manage Ubuntu kernel .config and annotations
# Copyright © 2022 Canonical Ltd.
import sys
import os
import argparse
import json
from signal import signal, SIGPIPE, SIG_DFL
try:
from argcomplete import autocomplete
except ModuleNotFoundError:
# Allow to run this program also when argcomplete is not available
def autocomplete(_unused):
pass
from kconfig.annotations import Annotation, KConfig # noqa: E402 Import not at top of file
from kconfig.utils import autodetect_annotations, arg_fail # noqa: E402 Import not at top of file
from kconfig.version import VERSION, ANNOTATIONS_FORMAT_VERSION # noqa: E402 Import not at top of file
SKIP_CONFIGS = (
# CONFIG_VERSION_SIGNATURE is dynamically set during the build
"CONFIG_VERSION_SIGNATURE",
# Allow to use a different versions of toolchain tools
"CONFIG_GCC_VERSION",
"CONFIG_CC_VERSION_TEXT",
"CONFIG_AS_VERSION",
"CONFIG_LD_VERSION",
"CONFIG_LLD_VERSION",
"CONFIG_CLANG_VERSION",
"CONFIG_PAHOLE_VERSION",
"CONFIG_RUSTC_VERSION_TEXT",
"CONFIG_BINDGEN_VERSION_TEXT",
)
def make_parser():
parser = argparse.ArgumentParser(
description="Manage Ubuntu kernel .config and annotations",
)
parser.add_argument("--version", "-v", action="version", version=f"%(prog)s {VERSION}")
parser.add_argument(
"--file",
"-f",
action="store",
help="Pass annotations or .config file to be parsed",
)
parser.add_argument("--arch", "-a", action="store", help="Select architecture")
parser.add_argument("--flavour", "-l", action="store", help='Select flavour (default is "generic")')
parser.add_argument("--config", "-c", action="store", help="Select a specific config option")
parser.add_argument("--query", "-q", action="store_true", help="Query annotations")
parser.add_argument(
"--note",
"-n",
action="store",
help="Write a specific note to a config option in annotations",
)
parser.add_argument(
"--autocomplete",
action="store_true",
help="Enable config bash autocomplete: `source <(annotations --autocomplete)`",
)
parser.add_argument(
"--source",
"-t",
action="store_true",
help="Jump to a config definition in the kernel source code",
)
parser.add_argument(
"--no-include",
action="store_true",
help="Do not process included annotations (stop at the main file)",
)
ga = parser.add_argument_group(title="Action").add_mutually_exclusive_group(required=False)
ga.add_argument(
"--write",
"-w",
action="store",
metavar="VALUE",
dest="value",
help="Set a specific config value in annotations (use 'null' to remove)",
)
ga.add_argument(
"--export",
"-e",
action="store_true",
help="Convert annotations to .config format",
)
ga.add_argument(
"--import",
"-i",
action="store",
metavar="FILE",
dest="import_file",
help="Import a full .config for a specific arch and flavour into annotations",
)
ga.add_argument(
"--update",
"-u",
action="store",
metavar="FILE",
dest="update_file",
help="Import a partial .config into annotations (only resync configs specified in FILE)",
)
ga.add_argument(
"--check",
"-k",
action="store",
metavar="FILE",
dest="check_file",
help="Validate kernel .config with annotations",
)
return parser
_ARGPARSER = make_parser()
def export_result(data):
# Dump metadata / attributes first
out = '{\n "attributes": {\n'
for key, value in sorted(data["attributes"].items()):
out += f' "{key}": {json.dumps(value)},\n'
out = out.rstrip(",\n")
out += "\n },"
print(out)
configs_with_note = {key: value for key, value in data["config"].items() if "note" in value}
configs_without_note = {key: value for key, value in data["config"].items() if "note" not in value}
# Dump configs, sorted alphabetically, showing items with a note first
out = ' "config": {\n'
for key in sorted(configs_with_note) + sorted(configs_without_note):
policy = data["config"][key]["policy"]
if "note" in data["config"][key]:
note = data["config"][key]["note"]
out += f' "{key}": {{"policy": {json.dumps(policy)}, "note": {json.dumps(note)}}},\n'
else:
out += f' "{key}": {{"policy": {json.dumps(policy)}}},\n'
out = out.rstrip(",\n")
out += "\n }\n}"
print(out)
def print_result(config, data):
if data is not None and config is not None and config not in data:
data = {config: data}
print(json.dumps(data, sort_keys=True, indent=2))
def do_query(args):
if args.arch is None and args.flavour is not None:
arg_fail(_ARGPARSER, "error: --flavour requires --arch")
a = Annotation(args.file, do_include=(not args.no_include))
res = a.search_config(config=args.config, arch=args.arch, flavour=args.flavour)
# If no arguments are specified dump the whole annotations structure
if args.config is None and args.arch is None and args.flavour is None:
res = {
"attributes": {
"arch": a.arch,
"flavour": a.flavour,
"flavour_dep": a.flavour_dep,
"include": a.include,
"_version": ANNOTATIONS_FORMAT_VERSION,
},
"config": res,
}
export_result(res)
else:
print_result(args.config, res)
def do_autocomplete(args):
a = Annotation(args.file)
res = (c.removeprefix("CONFIG_") for c in a.search_config())
res_str = " ".join(res)
print(f'complete -W "{res_str}" annotations')
def do_source(args):
if args.config is None:
arg_fail(_ARGPARSER, "error: --source requires --config")
if not os.path.exists("tags"):
print("tags not found in the current directory, try: `make tags`")
sys.exit(1)
os.system(f"vim -t {args.config}")
def do_note(args):
if args.config is None:
arg_fail(_ARGPARSER, "error: --note requires --config")
# Set the note in annotations
a = Annotation(args.file)
a.set(args.config, note=args.note)
# Save back to annotations
a.save(args.file)
# Query and print back the value
a = Annotation(args.file)
res = a.search_config(config=args.config)
print_result(args.config, res)
def do_write(args):
if args.config is None:
arg_fail(_ARGPARSER, "error: --write requires --config")
# Set the value in annotations ('null' means remove)
a = Annotation(args.file)
if args.value == "null":
a.remove(args.config, arch=args.arch, flavour=args.flavour)
else:
a.set(
args.config,
arch=args.arch,
flavour=args.flavour,
value=args.value,
note=args.note,
)
# Save back to annotations
a.save(args.file)
# Query and print back the value
a = Annotation(args.file)
res = a.search_config(config=args.config)
print_result(args.config, res)
def do_export(args):
if args.arch is None:
arg_fail(_ARGPARSER, "error: --export requires --arch")
a = Annotation(args.file)
conf = a.search_config(config=args.config, arch=args.arch, flavour=args.flavour)
if conf:
print(a.to_config(conf))
def do_import(args):
if args.arch is None:
arg_fail(_ARGPARSER, "error: --arch is required with --import")
if args.flavour is None:
arg_fail(_ARGPARSER, "error: --flavour is required with --import")
if args.config is not None:
arg_fail(_ARGPARSER, "error: --config cannot be used with --import (try --update)")
# Merge with the current annotations
a = Annotation(args.file)
c = KConfig(args.import_file)
a.update(c, arch=args.arch, flavour=args.flavour)
# Save back to annotations
a.save(args.file)
def do_update(args):
if args.arch is None:
arg_fail(_ARGPARSER, "error: --arch is required with --update")
# Merge with the current annotations
a = Annotation(args.file)
c = KConfig(args.update_file)
if args.config is None:
configs = list(set(c.config.keys()) - set(SKIP_CONFIGS))
if configs:
a.update(c, arch=args.arch, flavour=args.flavour, configs=configs)
# Save back to annotations
a.save(args.file)
def do_check(args):
# Determine arch and flavour
if args.arch is None:
arg_fail(_ARGPARSER, "error: --arch is required with --check")
print(f"check-config: loading annotations from {args.file}")
total = good = ret = 0
# Load annotations settings
a = Annotation(args.file)
a_configs = a.search_config(arch=args.arch, flavour=args.flavour).keys()
# Parse target .config
c = KConfig(args.check_file)
c_configs = c.config.keys()
# Validate .config against annotations
for conf in sorted(a_configs | c_configs):
if conf in SKIP_CONFIGS:
continue
entry = a.search_config(config=conf, arch=args.arch, flavour=args.flavour)
expected = entry[conf] if entry else "-"
value = c.config[conf] if conf in c.config else "-"
if value != expected:
policy = a.config[conf] if conf in a.config else "undefined"
if "policy" in policy:
policy = f"policy<{policy['policy']}>"
print(f"check-config: {conf} changed from {expected} to {value}: {policy})")
ret = 1
else:
good += 1
total += 1
num = total - good
if ret:
if os.path.exists(".git"):
print(f"check-config: {num} config options have been changed, review them with `git diff`")
else:
print(f"check-config: {num} config options have changed")
else:
print("check-config: all good")
sys.exit(ret)
def main():
# Prevent broken pipe errors when showing output in pipe to other tools
# (less for example)
signal(SIGPIPE, SIG_DFL)
# Main annotations program
autocomplete(_ARGPARSER)
args = _ARGPARSER.parse_args()
if args.file is None:
args.file = autodetect_annotations()
if args.file is None:
arg_fail(
_ARGPARSER,
"error: could not determine DEBDIR, try using: --file/-f",
show_usage=False,
)
if args.config and not args.config.startswith("CONFIG_"):
args.config = "CONFIG_" + args.config
if args.value:
do_write(args)
elif args.note:
do_note(args)
elif args.export:
do_export(args)
elif args.import_file:
do_import(args)
elif args.update_file:
do_update(args)
elif args.check_file:
do_check(args)
elif args.autocomplete:
do_autocomplete(args)
elif args.source:
do_source(args)
else:
do_query(args)
if __name__ == "__main__":
main()

20
debian/scripts/misc/kconfig/utils.py vendored Normal file
View File

@ -0,0 +1,20 @@
# -*- mode: python -*-
# Misc helpers for Kconfig and annotations
# Copyright © 2023 Canonical Ltd.
import sys
def autodetect_annotations():
try:
with open("debian/debian.env", "rt", encoding="utf-8") as fd:
return fd.read().rstrip().split("=")[1] + "/config/annotations"
except (FileNotFoundError, IndexError):
return None
def arg_fail(parser, message, show_usage=True):
print(message)
if show_usage:
parser.print_usage()
sys.exit(1)

10
debian/scripts/misc/kconfig/version.py vendored Normal file
View File

@ -0,0 +1,10 @@
# -*- mode: python -*-
# version of annotations module
# Copyright © 2022 Canonical Ltd.
VERSION = "0.1"
ANNOTATIONS_FORMAT_VERSION = 5
if __name__ == "__main__":
print(VERSION)