From 0f1d154a3af40a82626fbfecc7bee7c10a887cb5 Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Wed, 13 Jul 2016 13:26:55 +0200 Subject: [PATCH 1/2] oelite: add show-provided subcommand It can be frustrating trying to help track down problems which end up being caused by the other guy using a different autotools version. So "oe show-provided" does a semi-automatic extraction of the version of all ASSUME_PROVIDED utils. In many cases, we can just do "$util --version" and extract the first substring matching "\b([0-9][0-9.a-zA-Z_-]+)\b" - when necessary, this heuristic can be overridden, and I've included a few examples. We can now also attach minimum and maximum versions to the assume_provided utilities. We're not going to check or enforce these during normal "oe bake", but it's nice to be able to do "oe show-provided -c". A previous version had these minimum and maximum versions hardcoded in show_provided.py, but it's better to keep the version requirements together with the ASSUME_PROVIDED variable. The syntax is rather oelitish, e.g. ASSUME_PROVIDED_MIN += "native:git_1.8.4 native:tar_1.2.3" --- lib/oelite/cmd/__init__.py | 2 +- lib/oelite/cmd/show_provided.py | 185 ++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 lib/oelite/cmd/show_provided.py diff --git a/lib/oelite/cmd/__init__.py b/lib/oelite/cmd/__init__.py index 57090676..9f094afb 100644 --- a/lib/oelite/cmd/__init__.py +++ b/lib/oelite/cmd/__init__.py @@ -1 +1 @@ -manifest_cmds = [ "bake", "setup", "show", "cherry", "autodoc", "add-layer" ] +manifest_cmds = [ "bake", "setup", "show", "cherry", "autodoc", "add-layer", "show-provided" ] diff --git a/lib/oelite/cmd/show_provided.py b/lib/oelite/cmd/show_provided.py new file mode 100644 index 00000000..7804d07e --- /dev/null +++ b/lib/oelite/cmd/show_provided.py @@ -0,0 +1,185 @@ +import oebakery +import oelite.baker +import logging +from oelite.parse import confparse +import re +import subprocess +import bb.utils + +description = "Detect and print versions of provided utils" + +def add_parser_options(parser): + oelite.baker.add_show_parser_options(parser) + parser.add_option("-v", "--verbose", + action="store_true", + help="Be more chatty") + parser.add_option("-c", "--check", + action="store_true", + help="Check detected version against required min and max") + return + +def parse_args(options, args): + if options.verbose: + logging.getLogger().setLevel(logging.DEBUG) + else: + logging.getLogger().setLevel(logging.INFO) + +INITIAL_OE_IMPORTS = "sys os time" + +# One must describe how to get the version of each util. Fortunately, +# many utilities support a simple "--version" flag, and for most, the +# command to call is the same as what's in the ASSUME_PROVIDED (an +# exception is native:pkgconfig, where the command is pkg-config). So +# we don't need to be very verbose. +# +# extract: a regexp string which is matched against the output from +# the command. Should contain a single capture group, from which we'll +# get the version string +# +# ret: Some utilities exit with a non-zero value when asked to print +# their version (sometimes because they don't actually explicitly +# support --version or any other flag, but they display their version +# nevertheless as a side effect of telling the user he did something +# wrong...). This can be set to an integer or list of integers of +# expected return values. +data = dict() +data["native:binutils"] = dict(cmd = "ld") +data["native:coreutils"] = dict(cmd = "ls") +data["native:mercurial"] = dict(cmd = "hg") +data["native:mtd-utils-mkfs-jffs2"] = dict(cmd = "mkfs.jffs2", ret = 255) +data["native:perl"] = dict(extract = r"\(v([0-9][0-9.a-zA-Z_-]+)\)") +data["native:pkgconfig"] = dict(cmd = "pkg-config") +data["native:python-runtime"] = dict(cmd = "python") +data["native:texinfo"] = dict(cmd = "texindex") +data["native:unzip"] = dict(args = "-v") +data["native:util-linux"] = dict(cmd = "getopt") + +def fill_defaults(data, assume_provided): + # Provide an empty dict for those where we don't have any + # requirements. That makes it easy for us to (try to) run the + # default "get the version provided", possibly fixing up those + # where that fails. + for name in assume_provided: + if name in data: + continue + # Libraries would need to be handled in some other way... + if name.startswith("native:lib"): + continue + data[name] = dict() + for name in data: + param = data[name] + if not "cmd" in param: + if ":" in name: + param["cmd"] = name.split(":", 1)[1] + else: + param["cmd"] = name + + if not "args" in param: + param["args"] = "--version" + if isinstance(param["args"], basestring): + param["args"] = [param["args"]] + + if not "extract" in param: + param["extract"] = r"\b([0-9][0-9.a-zA-Z_-]+)\b" + + if not "ret" in param: + param["ret"] = 0 + if (isinstance(param["ret"], int)): + param["ret"] = [param["ret"]] + +def fill_versions(data, key, versions): + for util in versions: + u, v = util.split("_", 1) + if u in data: + data[u][key] = v + +def get_provided_version(p): + param = data.get(p) + if not param: + return None + + args = [param["cmd"]] + param["args"] + cmdstring = " ".join(args) + try: + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except OSError as e: + logging.warn("%s: failed to execute '%s': %s", p, cmdstring, e) + return None + output = process.communicate()[0] + + if process.returncode not in param["ret"]: + logging.warn("%s: command '%s' exited with status %d, expected one of %s" % + (p, cmdstring, process.returncode, str(param["ret"]))) + if output: + logging.debug("\n".join(map(lambda x: " " + x, output.split("\n")))) + return None + + match = re.search(param["extract"], output) + if not match: + logging.info("%s: regexp '%s' did not match output") + if output: + logging.warn("\n".join(map(lambda x: " " + x, output.split("\n")))) + return None + return match.group(1) + +def show_provided(p): + version = get_provided_version(p) + if version: + logging.info("%s: found version %s", p, version) + return None + +def check_provided(p): + param = data.get(p) + if not param or (not param.get("min") and not param.get("max")): + logging.debug("%s: no version requirements defined" % p) + return None + logging.debug("checking %s" % p) + + version = get_provided_version(p) + + logging.debug("%s: found version %s", p, version) + minversion = param.get("min") + maxversion = param.get("max") + ok = True + if minversion: + if bb.utils.vercmp_string(version, param["min"]) < 0: + logging.warn("%s: has version %s, expected minumum %s", p, version, minversion) + ok = False + if maxversion: + if bb.utils.vercmp_string(version, param["max"]) > 0: + logging.warn("%s: has version %s, expected maximum %s", p, version, maxversion) + ok = False + + return ok + +def run(options, args, config): + ret = 0 + config = oelite.meta.DictMeta(config) + config["OE_IMPORTS"] = INITIAL_OE_IMPORTS + config.pythonfunc_init() + + confparser = confparse.ConfParser(config) + confparser.parse("conf/oe-lite.conf") + + assume_provided = sorted(config.get("ASSUME_PROVIDED").split()) + fill_defaults(data, assume_provided) + assume_min = (config.get("ASSUME_PROVIDED_MIN") or "").split() + fill_versions(data, "min", assume_min) + assume_max = (config.get("ASSUME_PROVIDED_MAX") or "").split() + fill_versions(data, "max", assume_max) + + func = show_provided + if options.check: + func = check_provided + + for p in assume_provided: + ok = func(p) + if ok is None: + continue + if ok: + logging.info("%s: OK" % p) + else: + logging.info("%s: not OK" % p) + ret = 1 + + return ret From b2fb6f7c5e943680def6ee3e074dabfd34a34627 Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Fri, 21 Oct 2016 14:21:29 +0200 Subject: [PATCH 2/2] minimal.conf: assume git >= 1.8.4 This is partly to introduce an example of ASSUME_PROVIDED_MIN. But this is not just some random git version - I've had a difficult-to-debug problem which wouldn't have happened with a new enough host git. The problem was a SRC_URI specifying a git repository accessed via a non-standard ssh port, causing the ingredients subdir to contain a colon. Subsequent cloning from that dir (during do_unpack) was, pre-1.8.4, interpreted as an scp-style url rather than a local path, which obviously breaks. Commit 60003340cda05f5ecd79ee8522b21eda038b994b in git.git handles this case much more sanely. --- conf/oe-lite.conf | 1 + conf/provided/minimal.conf | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/conf/oe-lite.conf b/conf/oe-lite.conf index 6a9b41cb..d89d908b 100644 --- a/conf/oe-lite.conf +++ b/conf/oe-lite.conf @@ -48,6 +48,7 @@ LD_LIBRARY_PATH_VAR = "LD_LIBRARY_PATH" LD_LIBRARY_PATH_VAR:BUILD_KERNEL_darwin = "DYLD_LIBRARY_PATH" BLACKLIST_VAR = "BLACKLIST_VAR BLACKLIST_PREFIX" +BLACKLIST_VAR += "ASSUME_PROVIDED_MIN ASSUME_PROVIDED_MAX" # Variables which are only used by the initialization code and which # should not get inherited down to layer/recipe/task metadata. diff --git a/conf/provided/minimal.conf b/conf/provided/minimal.conf index 17367c07..00db060a 100644 --- a/conf/provided/minimal.conf +++ b/conf/provided/minimal.conf @@ -21,3 +21,7 @@ native:shasum \ native:gawk native:sed native:grep \ native:wget \ " + +ASSUME_PROVIDED_MIN = "\ +native:git_1.8.4 \ +"