--- /dev/null
+#!/usr/bin/env python
+
+# Copyright (C) 2014 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+# This script generates a random configuration for testing Buildroot.
+
+from __future__ import print_function
+
+import contextlib
+import csv
+import os
+from random import randint
+import subprocess
+import sys
+from time import localtime, strftime
+from distutils.version import StrictVersion
+import platform
+
+if sys.hexversion >= 0x3000000:
+ import urllib.request as _urllib
+else:
+ import urllib2 as _urllib
+
+urlopen = _urllib.urlopen
+urlopen_closing = lambda uri: contextlib.closing(urlopen(uri))
+
+if sys.hexversion >= 0x3000000:
+ def decode_byte_list(bl):
+ return [b.decode() for b in bl]
+else:
+ def decode_byte_list(e):
+ return e
+
+def log_write(logf, msg):
+ logf.write("[%s] %s\n" % (strftime("%a, %d %b %Y %H:%M:%S", localtime()), msg))
+ logf.flush()
+
+class SystemInfo:
+ DEFAULT_NEEDED_PROGS = ["make", "git", "gcc", "timeout"]
+ DEFAULT_OPTIONAL_PROGS = ["bzr", "java", "javac", "jar"]
+
+ def __init__(self):
+ self.needed_progs = list(self.__class__.DEFAULT_NEEDED_PROGS)
+ self.optional_progs = list(self.__class__.DEFAULT_OPTIONAL_PROGS)
+ self.progs = {}
+
+ def find_prog(self, name, flags=os.X_OK, env=os.environ):
+ if not name or name[0] == os.sep: raise ValueError(name)
+
+ prog_path = env.get("PATH", None)
+ # for windows compatibility, we'd need to take PATHEXT into account
+
+ if prog_path:
+ for prog_dir in filter(None, prog_path.split(os.pathsep)):
+ # os.join() not necessary: non-empty prog_dir
+ # and name[0] != os.sep
+ prog = prog_dir + os.sep + name
+ if os.access(prog, flags):
+ return prog
+ # --
+ return None
+
+ def has(self, prog):
+ """Checks whether a program is available.
+ Lazily evaluates missing entries.
+
+ Returns: None if prog not found, else path to the program [evaluates to True]
+ """
+ try:
+ return self.progs[prog]
+ except KeyError:
+ pass
+
+ have_it = self.find_prog(prog)
+ # java[c] needs special care
+ if have_it and prog in ('java', 'javac'):
+ with open(os.devnull, "w") as devnull:
+ if subprocess.call("%s -version | grep gcj" % prog, shell=True,
+ stdout=devnull, stderr=devnull) != 1:
+ have_it = False
+ # --
+ self.progs[prog] = have_it
+ return have_it
+
+ def check_requirements(self):
+ """Checks program dependencies.
+
+ Returns: True if all mandatory programs are present, else False.
+ """
+ do_check_has_prog = self.has
+
+ missing_requirements = False
+ for prog in self.needed_progs:
+ if not do_check_has_prog(prog):
+ print("ERROR: your system lacks the '%s' program" % prog)
+ missing_requirements = True
+
+ # check optional programs here,
+ # else they'd get checked by each worker instance
+ for prog in self.optional_progs:
+ do_check_has_prog(prog)
+
+ return not missing_requirements
+
+def get_toolchain_configs(**kwargs):
+ """Fetch and return the possible toolchain configurations
+
+ This function returns an array of toolchain configurations. Each
+ toolchain configuration is itself an array of lines of the defconfig.
+ """
+ toolchains_url = kwargs['toolchains_url']
+
+ with urlopen_closing(toolchains_url) as r:
+ l = decode_byte_list(r.readlines())
+ configs = []
+
+ (_, _, _, _, hostarch) = os.uname()
+ # ~2015 distros report x86 when on a 32bit install
+ if hostarch == 'i686' or hostarch == 'i386' or hostarch == 'x86':
+ hostarch = 'x86'
+
+ for row in csv.reader(l):
+ config = {}
+ url = row[0]
+ config_hostarch = row[1]
+ keep = False
+
+ # Keep all toolchain configs that work regardless of the host
+ # architecture
+ if config_hostarch == "any":
+ keep = True
+
+ # Keep all toolchain configs that can work on the current host
+ # architecture
+ if hostarch == config_hostarch:
+ keep = True
+
+ # Assume that x86 32 bits toolchains work on x86_64 build
+ # machines
+ if hostarch == 'x86_64' and config_hostarch == "x86":
+ keep = True
+
+ if not keep:
+ continue
+
+ with urlopen_closing(url) as r:
+ config = decode_byte_list(r.readlines())
+ configs.append(config)
+ return configs
+
+def is_toolchain_usable(**kwargs):
+ """Check if the toolchain is actually usable."""
+
+ idir = "instance-%d" % kwargs['instance']
+ sysinfo = kwargs['sysinfo']
+ log = kwargs['log']
+
+ outputdir = os.path.join(idir, "output")
+ with open(os.path.join(outputdir, ".config")) as configf:
+ configlines = configf.readlines()
+
+ # Check that the toolchain configuration is still present
+ for toolchainline in kwargs['config']:
+ if toolchainline not in configlines:
+ return False
+
+ # The latest Linaro toolchains on x86-64 hosts requires glibc
+ # 2.14+ on the host.
+ if platform.machine() == 'x86_64':
+ if 'BR2_TOOLCHAIN_EXTERNAL_LINARO_ARM=y\n' in configlines or \
+ 'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64=y\n' in configlines or \
+ 'BR2_TOOLCHAIN_EXTERNAL_LINARO_ARMEB=y\n' in configlines:
+ ldd_version_output = subprocess.Popen(['ldd', '--version'], stdout=subprocess.PIPE).communicate()[0]
+ glibc_version = ldd_version_output.splitlines()[0].split()[-1]
+ if StrictVersion('2.14') > StrictVersion(glibc_version):
+ log_write(log, "WARN: ignoring the Linaro ARM toolchains becausee too old host glibc")
+ return False
+
+ return True
+
+def fixup_config(**kwargs):
+ """Finalize the configuration and reject any problematic combinations
+
+ This function returns 'True' when the configuration has been
+ accepted, and 'False' when the configuration has not been accepted because
+ it is known to fail (in which case another random configuration will be
+ generated).
+ """
+
+ idir = "instance-%d" % kwargs['instance']
+ sysinfo = kwargs['sysinfo']
+
+ outputdir = os.path.join(idir, "output")
+ with open(os.path.join(outputdir, ".config")) as configf:
+ configlines = configf.readlines()
+
+ if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not sysinfo.has("java"):
+ return False
+ if "BR2_NEEDS_HOST_JAVAC=y\n" in configlines and not sysinfo.has("javac"):
+ return False
+ if "BR2_NEEDS_HOST_JAR=y\n" in configlines and not sysinfo.has("jar"):
+ return False
+ # python-nfc needs bzr
+ if 'BR2_PACKAGE_PYTHON_NFC=y\n' in configlines and not sysinfo.has("bzr"):
+ return False
+ # The ctng toolchain is affected by PR58854
+ if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \
+ 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/armv5-ctng-linux-gnueabi.tar.xz"\n' in configlines:
+ return False
+ # The ctng toolchain tigger an assembler error with guile package when compiled with -Os (same issue as for CS ARM 2014.05-29)
+ if 'BR2_PACKAGE_GUILE=y\n' in configlines and 'BR2_OPTIMIZE_S=y\n' in configlines and \
+ 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/armv5-ctng-linux-gnueabi.tar.xz"\n' in configlines:
+ return False
+ # The ctng toolchain is affected by PR58854
+ if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \
+ 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/armv6-ctng-linux-uclibcgnueabi.tar.xz"\n' in configlines:
+ return False
+ # The ctng toolchain is affected by PR58854
+ if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \
+ 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/armv7-ctng-linux-gnueabihf.tar.xz"\n' in configlines:
+ return False
+ # The ctng toolchain is affected by PR60155
+ if 'BR2_PACKAGE_SDL=y\n' in configlines and \
+ 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/powerpc-ctng-linux-uclibc.tar.xz"\n' in configlines:
+ return False
+ # The ctng toolchain is affected by PR60155
+ if 'BR2_PACKAGE_LIBMPEG2=y\n' in configlines and \
+ 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/powerpc-ctng-linux-uclibc.tar.xz"\n' in configlines:
+ return False
+ # This MIPS toolchain uses eglibc-2.18 which lacks SYS_getdents64
+ if 'BR2_PACKAGE_STRONGSWAN=y\n' in configlines and \
+ 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/mips64el-ctng_n64-linux-gnu.tar.xz"\n' in configlines:
+ return False
+ # This MIPS toolchain uses eglibc-2.18 which lacks SYS_getdents64
+ if 'BR2_PACKAGE_PYTHON3=y\n' in configlines and \
+ 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/mips64el-ctng_n64-linux-gnu.tar.xz"\n' in configlines:
+ return False
+ # libffi not available on sh2a and ARMv7-M, but propagating libffi
+ # arch dependencies in Buildroot is really too much work, so we
+ # handle this here.
+ if 'BR2_sh2a=y\n' in configlines and 'BR2_PACKAGE_LIBFFI=y\n' in configlines:
+ return False
+ if 'BR2_ARM_CPU_ARMV7M=y\n' in configlines and 'BR2_PACKAGE_LIBFFI=y\n' in configlines:
+ return False
+ if 'BR2_PACKAGE_SUNXI_BOARDS=y\n' in configlines:
+ configlines.remove('BR2_PACKAGE_SUNXI_BOARDS_FEX_FILE=""\n')
+ configlines.append('BR2_PACKAGE_SUNXI_BOARDS_FEX_FILE="a10/hackberry.fex"\n')
+ # This MIPS uClibc toolchain fails to build the gdb package
+ if 'BR2_PACKAGE_GDB=y\n' in configlines and \
+ 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines:
+ return False
+ # This MIPS uClibc toolchain fails to build the rt-tests package
+ if 'BR2_PACKAGE_RT_TESTS=y\n' in configlines and \
+ 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines:
+ return False
+ # This MIPS uClibc toolchain fails to build the civetweb package
+ if 'BR2_PACKAGE_CIVETWEB=y\n' in configlines and \
+ 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines:
+ return False
+ # This MIPS ctng toolchain fails to build the python3 package
+ if 'BR2_PACKAGE_PYTHON3=y\n' in configlines and \
+ 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/mips64el-ctng_n64-linux-gnu.tar.xz"\n' in configlines:
+ return False
+ # This MIPS uClibc toolchain fails to build the strace package
+ if 'BR2_PACKAGE_STRACE=y\n' in configlines and \
+ 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines:
+ return False
+ # This MIPS uClibc toolchain fails to build the cdrkit package
+ if 'BR2_PACKAGE_CDRKIT=y\n' in configlines and \
+ 'BR2_STATIC_LIBS=y\n' in configlines and \
+ 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines:
+ return False
+ # uClibc vfork static linking issue
+ if 'BR2_PACKAGE_ALSA_LIB=y\n' in configlines and \
+ 'BR2_STATIC_LIBS=y\n' in configlines and \
+ 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/i486-ctng-linux-uclibc.tar.xz"\n' in configlines:
+ return False
+ # This MIPS uClibc toolchain fails to build the weston package
+ if 'BR2_PACKAGE_WESTON=y\n' in configlines and \
+ 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines:
+ return False
+ # The cs nios2 2017.02 toolchain is affected by binutils PR19405
+ if 'BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_NIOSII=y\n' in configlines and \
+ 'BR2_PACKAGE_BOOST=y\n' in configlines:
+ return False
+ # The cs nios2 2017.02 toolchain is affected by binutils PR19405
+ if 'BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_NIOSII=y\n' in configlines and \
+ 'BR2_PACKAGE_QT5BASE_GUI=y\n' in configlines:
+ return False
+ # The cs nios2 2017.02 toolchain is affected by binutils PR19405
+ if 'BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_NIOSII=y\n' in configlines and \
+ 'BR2_PACKAGE_QT_GUI_MODULE=y\n' in configlines:
+ return False
+ # The cs nios2 2017.02 toolchain is affected by binutils PR19405
+ if 'BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_NIOSII=y\n' in configlines and \
+ 'BR2_PACKAGE_FLANN=y\n' in configlines:
+ return False
+ # or1k affected by binutils PR21464
+ if 'BR2_or1k=y\n' in configlines and \
+ 'BR2_PACKAGE_QT_GUI_MODULE=y\n' in configlines:
+ return False
+
+ with open(os.path.join(outputdir, ".config"), "w+") as configf:
+ configf.writelines(configlines)
+
+ return True
+
+def gen_config(**kwargs):
+ """Generate a new random configuration
+
+ This function generates the configuration, by choosing a random
+ toolchain configuration and then generating a random selection of
+ packages.
+ """
+
+ idir = "instance-%d" % kwargs['instance']
+ log = kwargs['log']
+
+ # We need the absolute path to use with O=, because the relative
+ # path to the output directory here is not relative to the
+ # Buildroot sources, but to the location of the autobuilder
+ # script.
+ outputdir = os.path.abspath(os.path.join(idir, "output"))
+ srcdir = os.path.join(idir, "buildroot")
+
+ log_write(log, "INFO: generate the configuration")
+
+ # Select a random toolchain configuration
+ try:
+ configs = get_toolchain_configs(**kwargs)
+ except:
+ return -1
+
+ i = randint(0, len(configs) - 1)
+ config = configs[i]
+
+ configlines = config
+
+ # Amend the configuration with a few things.
+ configlines.append("BR2_PACKAGE_BUSYBOX_SHOW_OTHERS=y\n")
+ configlines.append("# BR2_TARGET_ROOTFS_TAR is not set\n")
+ configlines.append("BR2_COMPILER_PARANOID_UNSAFE_PATH=y\n")
+ if randint(0, 20) == 0:
+ configlines.append("BR2_ENABLE_DEBUG=y\n")
+ if randint(0, 30) == 0:
+ configlines.append("BR2_INIT_SYSTEMD=y\n")
+ elif randint(0, 20) == 0:
+ configlines.append("BR2_ROOTFS_DEVICE_CREATION_DYNAMIC_EUDEV=y\n")
+ if randint(0, 20) == 0:
+ configlines.append("BR2_STATIC_LIBS=y\n")
+ if randint(0, 20) == 0:
+ configlines.append("BR2_PACKAGE_PYTHON_PY_ONLY=y\n")
+
+ # Write out the configuration file
+ with open(os.path.join(outputdir, ".config"), "w+") as configf:
+ configf.writelines(configlines)
+
+ devnull = open(os.devnull, "w")
+
+ ret = subprocess.call(["make", "O=%s" % outputdir, "-C", srcdir, "olddefconfig"],
+ stdout=devnull, stderr=devnull)
+ if ret != 0:
+ log_write(log, "ERROR: cannot oldconfig")
+ return -1
+
+ if not is_toolchain_usable(config=config, **kwargs):
+ return -1
+
+ # Now, generate the random selection of packages, and fixup
+ # things if needed.
+ # Safe-guard, in case we can not quickly come to a valid
+ # configuration: allow at most 100 (arbitrary) iterations.
+ bounded_loop = 100
+ while True:
+ if bounded_loop == 0:
+ log_write(log, "ERROR: cannot generate random configuration after 100 iterations")
+ return -1
+ bounded_loop -= 1
+ ret = subprocess.call(["make", "O=%s" % outputdir, "-C", srcdir,
+ "KCONFIG_PROBABILITY=%d" % randint(1,30), "randpackageconfig"],
+ stdout=devnull, stderr=devnull)
+ if ret != 0:
+ log_write(log, "ERROR: cannot generate random configuration")
+ return -1
+ if fixup_config(**kwargs):
+ break
+
+ ret = subprocess.call(["make", "O=%s" % outputdir, "-C", srcdir, "olddefconfig"],
+ stdout=devnull, stderr=devnull)
+ if ret != 0:
+ log_write(log, "ERROR: cannot oldconfig")
+ return -1
+
+ ret = subprocess.call(["make", "O=%s" % outputdir, "-C", srcdir, "savedefconfig"],
+ stdout=devnull, stderr=devnull)
+ if ret != 0:
+ log_write(log, "ERROR: cannot savedefconfig")
+ return -1
+
+ return 0
+
+
+if __name__ == '__main__':
+ import argparse
+ parser = argparse.ArgumentParser(description="Generate a random configuration")
+ parser.add_argument("--instance", "-i",
+ help="Instance number for creating unique directories",
+ type=int, default=0)
+ parser.add_argument("--toolchains-url",
+ help="URL of toolchain configuration file",
+ type=str,
+ default="http://autobuild.buildroot.org/toolchains/configs/toolchain-configs.csv")
+ args = parser.parse_args()
+
+ # Arguments expected by gen_config for which we just set a default here
+ args.log = sys.stdout
+ args.sysinfo = SystemInfo()
+
+ # Output directory is already created by autobuild-run so emulate it here
+ idir = "instance-%d" % args.instance
+ if not os.path.exists(idir):
+ os.mkdir(idir)
+ os.mkdir(os.path.join(idir, "output"))
+ # gen_config expects "buildroot" directory under idir
+ os.symlink("..", os.path.join(idir, "buildroot"))
+
+ # gen_config expects a dict, but args is a class object
+ ret = gen_config(**args.__dict__)
+
+ if ret != 0:
+ parser.exit(1)