3 # Copyright (C) 2014 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 2 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 # General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 # This script generates a random configuration for testing Buildroot.
21 from __future__ import print_function
26 from random import randint
29 from distutils.version import StrictVersion
32 if sys.hexversion >= 0x3000000:
33 import urllib.request as _urllib
35 import urllib2 as _urllib
38 def urlopen_closing(uri):
39 return contextlib.closing(_urllib.urlopen(uri))
42 if sys.hexversion >= 0x3000000:
43 def decode_byte_list(bl):
44 return [b.decode() for b in bl]
46 def decode_byte_list(e):
51 DEFAULT_NEEDED_PROGS = ["make", "git", "gcc", "timeout"]
52 DEFAULT_OPTIONAL_PROGS = ["bzr", "java", "javac", "jar"]
55 self.needed_progs = list(self.__class__.DEFAULT_NEEDED_PROGS)
56 self.optional_progs = list(self.__class__.DEFAULT_OPTIONAL_PROGS)
59 def find_prog(self, name, flags=os.X_OK, env=os.environ):
60 if not name or name[0] == os.sep:
61 raise ValueError(name)
63 prog_path = env.get("PATH", None)
64 # for windows compatibility, we'd need to take PATHEXT into account
67 for prog_dir in filter(None, prog_path.split(os.pathsep)):
68 # os.join() not necessary: non-empty prog_dir
69 # and name[0] != os.sep
70 prog = prog_dir + os.sep + name
71 if os.access(prog, flags):
77 """Checks whether a program is available.
78 Lazily evaluates missing entries.
80 Returns: None if prog not found, else path to the program [evaluates
84 return self.progs[prog]
88 have_it = self.find_prog(prog)
89 # java[c] needs special care
90 if have_it and prog in ('java', 'javac'):
91 with open(os.devnull, "w") as devnull:
92 if subprocess.call("%s -version | grep gcj" % prog,
94 stdout=devnull, stderr=devnull) != 1:
97 self.progs[prog] = have_it
100 def check_requirements(self):
101 """Checks program dependencies.
103 Returns: True if all mandatory programs are present, else False.
105 do_check_has_prog = self.has
107 missing_requirements = False
108 for prog in self.needed_progs:
109 if not do_check_has_prog(prog):
110 print("ERROR: your system lacks the '%s' program" % prog)
111 missing_requirements = True
113 # check optional programs here,
114 # else they'd get checked by each worker instance
115 for prog in self.optional_progs:
116 do_check_has_prog(prog)
118 return not missing_requirements
121 def get_toolchain_configs(toolchains_csv, buildrootdir):
122 """Fetch and return the possible toolchain configurations
124 This function returns an array of toolchain configurations. Each
125 toolchain configuration is itself an array of lines of the defconfig.
128 with open(toolchains_csv) as r:
129 # filter empty lines and comments
130 lines = [t for t in r.readlines() if len(t.strip()) > 0 and t[0] != '#']
131 toolchains = decode_byte_list(lines)
134 (_, _, _, _, hostarch) = os.uname()
135 # ~2015 distros report x86 when on a 32bit install
136 if hostarch == 'i686' or hostarch == 'i386' or hostarch == 'x86':
139 for row in csv.reader(toolchains):
142 config_hostarch = row[1]
145 # Keep all toolchain configs that work regardless of the host
147 if config_hostarch == "any":
150 # Keep all toolchain configs that can work on the current host
152 if hostarch == config_hostarch:
155 # Assume that x86 32 bits toolchains work on x86_64 build
157 if hostarch == 'x86_64' and config_hostarch == "x86":
163 if not os.path.isabs(configfile):
164 configfile = os.path.join(buildrootdir, configfile)
166 with open(configfile) as r:
167 config = r.readlines()
168 configs.append(config)
172 def is_toolchain_usable(configfile, config):
173 """Check if the toolchain is actually usable."""
175 with open(configfile) as configf:
176 configlines = configf.readlines()
178 # Check that the toolchain configuration is still present
179 for toolchainline in config:
180 if toolchainline not in configlines:
181 print("WARN: toolchain can't be used", file=sys.stderr)
182 print(" Missing: %s" % toolchainline.strip(), file=sys.stderr)
185 # The latest Linaro toolchains on x86-64 hosts requires glibc
187 if platform.machine() == 'x86_64':
188 if 'BR2_TOOLCHAIN_EXTERNAL_LINARO_ARM=y\n' in configlines or \
189 'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64=y\n' in configlines or \
190 'BR2_TOOLCHAIN_EXTERNAL_LINARO_ARMEB=y\n' in configlines:
191 ldd_version_output = subprocess.check_output(['ldd', '--version'])
192 glibc_version = ldd_version_output.splitlines()[0].split()[-1]
193 if StrictVersion('2.14') > StrictVersion(glibc_version):
194 print("WARN: ignoring the Linaro ARM toolchains because too old host glibc", file=sys.stderr)
200 def fixup_config(configfile):
201 """Finalize the configuration and reject any problematic combinations
203 This function returns 'True' when the configuration has been
204 accepted, and 'False' when the configuration has not been accepted because
205 it is known to fail (in which case another random configuration will be
209 sysinfo = SystemInfo()
210 with open(configfile) as configf:
211 configlines = configf.readlines()
213 BR2_TOOLCHAIN_EXTERNAL_URL = 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/'
215 if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not sysinfo.has("java"):
217 if "BR2_NEEDS_HOST_JAVAC=y\n" in configlines and not sysinfo.has("javac"):
219 if "BR2_NEEDS_HOST_JAR=y\n" in configlines and not sysinfo.has("jar"):
221 # python-nfc needs bzr
222 if 'BR2_PACKAGE_PYTHON_NFC=y\n' in configlines and not sysinfo.has("bzr"):
224 # The ctng toolchain is affected by PR58854
225 if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \
226 BR2_TOOLCHAIN_EXTERNAL_URL + 'armv5-ctng-linux-gnueabi.tar.xz"\n' in configlines:
228 # The ctng toolchain tigger an assembler error with guile package when compiled with -Os (same issue as for CS ARM 2014.05-29)
229 if 'BR2_PACKAGE_GUILE=y\n' in configlines and \
230 'BR2_OPTIMIZE_S=y\n' in configlines and \
231 BR2_TOOLCHAIN_EXTERNAL_URL + 'armv5-ctng-linux-gnueabi.tar.xz"\n' in configlines:
233 # The ctng toolchain is affected by PR58854
234 if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \
235 BR2_TOOLCHAIN_EXTERNAL_URL + 'armv6-ctng-linux-uclibcgnueabi.tar.xz"\n' in configlines:
237 # The ctng toolchain is affected by PR58854
238 if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \
239 BR2_TOOLCHAIN_EXTERNAL_URL + 'armv7-ctng-linux-gnueabihf.tar.xz"\n' in configlines:
241 # The ctng toolchain is affected by PR60155
242 if 'BR2_PACKAGE_SDL=y\n' in configlines and \
243 BR2_TOOLCHAIN_EXTERNAL_URL + 'powerpc-ctng-linux-uclibc.tar.xz"\n' in configlines:
245 # The ctng toolchain is affected by PR60155
246 if 'BR2_PACKAGE_LIBMPEG2=y\n' in configlines and \
247 BR2_TOOLCHAIN_EXTERNAL_URL + 'powerpc-ctng-linux-uclibc.tar.xz"\n' in configlines:
249 # This MIPS toolchain uses eglibc-2.18 which lacks SYS_getdents64
250 if 'BR2_PACKAGE_STRONGSWAN=y\n' in configlines and \
251 BR2_TOOLCHAIN_EXTERNAL_URL + 'mips64el-ctng_n64-linux-gnu.tar.xz"\n' in configlines:
253 # This MIPS toolchain uses eglibc-2.18 which lacks SYS_getdents64
254 if 'BR2_PACKAGE_PYTHON3=y\n' in configlines and \
255 BR2_TOOLCHAIN_EXTERNAL_URL + 'mips64el-ctng_n64-linux-gnu.tar.xz"\n' in configlines:
257 # libffi not available on sh2a and ARMv7-M, but propagating libffi
258 # arch dependencies in Buildroot is really too much work, so we
260 if 'BR2_sh2a=y\n' in configlines and \
261 'BR2_PACKAGE_LIBFFI=y\n' in configlines:
263 if 'BR2_ARM_CPU_ARMV7M=y\n' in configlines and \
264 'BR2_PACKAGE_LIBFFI=y\n' in configlines:
266 if 'BR2_PACKAGE_SUNXI_BOARDS=y\n' in configlines:
267 configlines.remove('BR2_PACKAGE_SUNXI_BOARDS_FEX_FILE=""\n')
268 configlines.append('BR2_PACKAGE_SUNXI_BOARDS_FEX_FILE="a10/hackberry.fex"\n')
269 # This MIPS uClibc toolchain fails to build the gdb package
270 if 'BR2_PACKAGE_GDB=y\n' in configlines and \
271 BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines:
273 # This MIPS uClibc toolchain fails to build the rt-tests package
274 if 'BR2_PACKAGE_RT_TESTS=y\n' in configlines and \
275 BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines:
277 # This MIPS uClibc toolchain fails to build the civetweb package
278 if 'BR2_PACKAGE_CIVETWEB=y\n' in configlines and \
279 BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines:
281 # This MIPS ctng toolchain fails to build the python3 package
282 if 'BR2_PACKAGE_PYTHON3=y\n' in configlines and \
283 BR2_TOOLCHAIN_EXTERNAL_URL + 'mips64el-ctng_n64-linux-gnu.tar.xz"\n' in configlines:
285 # This MIPS uClibc toolchain fails to build the strace package
286 if 'BR2_PACKAGE_STRACE=y\n' in configlines and \
287 BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines:
289 # This MIPS uClibc toolchain fails to build the cdrkit package
290 if 'BR2_PACKAGE_CDRKIT=y\n' in configlines and \
291 'BR2_STATIC_LIBS=y\n' in configlines and \
292 BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines:
294 # uClibc vfork static linking issue
295 if 'BR2_PACKAGE_ALSA_LIB=y\n' in configlines and \
296 'BR2_STATIC_LIBS=y\n' in configlines and \
297 BR2_TOOLCHAIN_EXTERNAL_URL + 'i486-ctng-linux-uclibc.tar.xz"\n' in configlines:
299 # This MIPS uClibc toolchain fails to build the weston package
300 if 'BR2_PACKAGE_WESTON=y\n' in configlines and \
301 BR2_TOOLCHAIN_EXTERNAL_URL + 'mipsel-ctng-linux-uclibc.tar.xz"\n' in configlines:
303 # The cs nios2 2017.02 toolchain is affected by binutils PR19405
304 if 'BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_NIOSII=y\n' in configlines and \
305 'BR2_PACKAGE_BOOST=y\n' in configlines:
307 # The cs nios2 2017.02 toolchain is affected by binutils PR19405
308 if 'BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_NIOSII=y\n' in configlines and \
309 'BR2_PACKAGE_QT5BASE_GUI=y\n' in configlines:
311 # The cs nios2 2017.02 toolchain is affected by binutils PR19405
312 if 'BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_NIOSII=y\n' in configlines and \
313 'BR2_PACKAGE_QT_GUI_MODULE=y\n' in configlines:
315 # The cs nios2 2017.02 toolchain is affected by binutils PR19405
316 if 'BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_NIOSII=y\n' in configlines and \
317 'BR2_PACKAGE_FLANN=y\n' in configlines:
319 # or1k affected by binutils PR21464
320 if 'BR2_or1k=y\n' in configlines and \
321 'BR2_PACKAGE_QT_GUI_MODULE=y\n' in configlines:
324 with open(configfile, "w+") as configf:
325 configf.writelines(configlines)
330 def gen_config(args):
331 """Generate a new random configuration
333 This function generates the configuration, by choosing a random
334 toolchain configuration and then generating a random selection of
338 # Select a random toolchain configuration
339 configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir)
341 i = randint(0, len(configs) - 1)
342 toolchainconfig = configs[i]
344 configlines = list(toolchainconfig)
346 # Combine with the minimal configuration
347 minimalconfigfile = os.path.join(args.buildrootdir,
348 'support/config-fragments/minimal.config')
349 with open(minimalconfigfile) as minimalf:
350 configlines += minimalf.readlines()
352 # Allow hosts with old certificates to download over https
353 configlines.append("BR2_WGET=\"wget --passive-ftp -nd -t 3 --no-check-certificate\"")
355 # Amend the configuration with a few things.
356 if randint(0, 20) == 0:
357 configlines.append("BR2_ENABLE_DEBUG=y\n")
358 if randint(0, 1) == 0:
359 configlines.append("BR2_INIT_BUSYBOX=y\n")
360 elif randint(0, 15) == 0:
361 configlines.append("BR2_INIT_SYSTEMD=y\n")
362 elif randint(0, 10) == 0:
363 configlines.append("BR2_ROOTFS_DEVICE_CREATION_DYNAMIC_EUDEV=y\n")
364 if randint(0, 20) == 0:
365 configlines.append("BR2_STATIC_LIBS=y\n")
366 if randint(0, 20) == 0:
367 configlines.append("BR2_PACKAGE_PYTHON_PY_ONLY=y\n")
369 # Write out the configuration file
370 if not os.path.exists(args.outputdir):
371 os.makedirs(args.outputdir)
372 if args.outputdir == os.path.abspath(os.path.join(args.buildrootdir, "output")):
373 configfile = os.path.join(args.buildrootdir, ".config")
375 configfile = os.path.join(args.outputdir, ".config")
376 with open(configfile, "w+") as configf:
377 configf.writelines(configlines)
379 subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
382 if not is_toolchain_usable(configfile, toolchainconfig):
385 # Now, generate the random selection of packages, and fixup
387 # Safe-guard, in case we can not quickly come to a valid
388 # configuration: allow at most 100 (arbitrary) iterations.
391 if bounded_loop == 0:
392 print("ERROR: cannot generate random configuration after 100 iterations",
396 subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
397 "KCONFIG_PROBABILITY=%d" % randint(1, 30),
398 "randpackageconfig"])
400 if fixup_config(configfile):
403 subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
406 subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
409 return subprocess.call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
413 if __name__ == '__main__':
415 parser = argparse.ArgumentParser(description="Generate a random configuration")
416 parser.add_argument("--outputdir", "-o",
417 help="Output directory (relative to current directory)",
418 type=str, default='output')
419 parser.add_argument("--buildrootdir", "-b",
420 help="Buildroot directory (relative to current directory)",
421 type=str, default='.')
422 parser.add_argument("--toolchains-csv",
423 help="Path of the toolchain configuration file",
425 default="support/config-fragments/autobuild/toolchain-configs.csv")
426 args = parser.parse_args()
428 # We need the absolute path to use with O=, because the relative
429 # path to the output directory here is not relative to the
430 # Buildroot sources, but to the current directory.
431 args.outputdir = os.path.abspath(args.outputdir)
434 ret = gen_config(args)
435 except Exception as e:
436 print(str(e), file=sys.stderr)