support/scripts/pkg-stats-new

   1 #!/usr/bin/env python
   2
   3 # Copyright (C) 2009 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 2 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  13 # General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program; if not, write to the Free Software
  17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  18
  19 import argparse
  20 import datetime
  21 import fnmatch
  22 import os
  23 from collections import defaultdict
  24 import re
  25 import subprocess
  26
  27 INFRA_RE = re.compile("\$\(eval \$\(([a-z-]*)-package\)\)")
  28
  29
  30 class Package:
  31     all_licenses = list()
  32     all_license_files = list()
  33
  34     def __init__(self, name, path):
  35         self.name = name
  36         self.path = path
  37         self.infras = None
  38         self.has_license = False
  39         self.has_license_files = False
  40         self.has_hash = False
  41         self.patch_count = 0
  42         self.warnings = 0
  43
  44     def pkgvar(self):
  45         return self.name.upper().replace("-", "_")
  46
  47     def set_infra(self):
  48         """
  49         Fills in the .infras field
  50         """
  51         self.infras = list()
  52         with open(self.path, 'r') as f:
  53             lines = f.readlines()
  54             for l in lines:
  55                 match = INFRA_RE.match(l)
  56                 if not match:
  57                     continue
  58                 infra = match.group(1)
  59                 if infra.startswith("host-"):
  60                     self.infras.append(("host", infra[5:]))
  61                 else:
  62                     self.infras.append(("target", infra))
  63
  64     def set_license(self):
  65         """
  66         Fills in the .has_license and .has_license_files fields
  67         """
  68         var = self.pkgvar()
  69         if var in self.all_licenses:
  70             self.has_license = True
  71         if var in self.all_license_files:
  72             self.has_license_files = True
  73
  74     def set_hash_info(self):
  75         """
  76         Fills in the .has_hash field
  77         """
  78         hashpath = self.path.replace(".mk", ".hash")
  79         self.has_hash = os.path.exists(hashpath)
  80
  81     def set_patch_count(self):
  82         """
  83         Fills in the .patch_count field
  84         """
  85         self.patch_count = 0
  86         pkgdir = os.path.dirname(self.path)
  87         for subdir, _, _ in os.walk(pkgdir):
  88             self.patch_count += len(fnmatch.filter(os.listdir(subdir), '*.patch'))
  89
  90     def set_check_package_warnings(self):
  91         """
  92         Fills in the .warnings field
  93         """
  94         cmd = ["./utils/check-package"]
  95         pkgdir = os.path.dirname(self.path)
  96         for root, dirs, files in os.walk(pkgdir):
  97             for f in files:
  98                 if f.endswith(".mk") or f.endswith(".hash") or f == "Config.in" or f == "Config.in.host":
  99                     cmd.append(os.path.join(root, f))
 100         o = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[1]
 101         lines = o.splitlines()
 102         for line in lines:
 103             m = re.match("^([0-9]*) warnings generated", line)
 104             if m:
 105                 self.warnings = int(m.group(1))
 106                 return
 107
 108     def __eq__(self, other):
 109         return self.path == other.path
 110
 111     def __lt__(self, other):
 112         return self.path < other.path
 113
 114     def __str__(self):
 115         return "%s (path='%s', license='%s', license_files='%s', hash='%s', patches=%d)" % \
 116             (self.name, self.path, self.has_license, self.has_license_files, self.has_hash, self.patch_count)
 117
 118
 119 def get_pkglist():
 120     """
 121     Builds the list of Buildroot packages, returning a list of Package
 122     objects. Only the .name and .path fields of the Package object are
 123     initialized.
 124     """
 125     WALK_USEFUL_SUBDIRS = ["boot", "linux", "package", "toolchain"]
 126     WALK_EXCLUDES = ["boot/common.mk",
 127                      "linux/linux-ext-.*.mk",
 128                      "package/freescale-imx/freescale-imx.mk",
 129                      "package/gcc/gcc.mk",
 130                      "package/gstreamer/gstreamer.mk",
 131                      "package/gstreamer1/gstreamer1.mk",
 132                      "package/gtk2-themes/gtk2-themes.mk",
 133                      "package/matchbox/matchbox.mk",
 134                      "package/opengl/opengl.mk",
 135                      "package/qt5/qt5.mk",
 136                      "package/x11r7/x11r7.mk",
 137                      "package/doc-asciidoc.mk",
 138                      "package/pkg-.*.mk",
 139                      "package/nvidia-tegra23/nvidia-tegra23.mk",
 140                      "toolchain/toolchain-external/pkg-toolchain-external.mk",
 141                      "toolchain/toolchain-external/toolchain-external.mk",
 142                      "toolchain/toolchain.mk",
 143                      "toolchain/helpers.mk",
 144                      "toolchain/toolchain-wrapper.mk"]
 145     packages = list()
 146     for root, dirs, files in os.walk("."):
 147         rootdir = root.split("/")
 148         if len(rootdir) < 2:
 149             continue
 150         if rootdir[1] not in WALK_USEFUL_SUBDIRS:
 151             continue
 152         for f in files:
 153             if not f.endswith(".mk"):
 154                 continue
 155             # Strip ending ".mk"
 156             pkgname = f[:-3]
 157             pkgpath = os.path.join(root, f)
 158             skip = False
 159             for exclude in WALK_EXCLUDES:
 160                 # pkgpath[2:] strips the initial './'
 161                 if re.match(exclude, pkgpath[2:]):
 162                     skip = True
 163                     continue
 164             if skip:
 165                 continue
 166             p = Package(pkgname, pkgpath)
 167             packages.append(p)
 168     return packages
 169
 170
 171 def package_init_make_info():
 172     # Licenses
 173     o = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y",
 174                                  "-s", "printvars", "VARS=%_LICENSE"])
 175     for l in o.splitlines():
 176         # Get variable name and value
 177         pkgvar, value = l.split("=")
 178
 179         # If present, strip HOST_ from variable name
 180         if pkgvar.startswith("HOST_"):
 181             pkgvar = pkgvar[5:]
 182
 183         # Strip _LICENSE
 184         pkgvar = pkgvar[:-8]
 185
 186         # If value is "unknown", no license details available
 187         if value == "unknown":
 188             continue
 189         Package.all_licenses.append(pkgvar)
 190
 191     # License files
 192     o = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y",
 193                                  "-s", "printvars", "VARS=%_LICENSE_FILES"])
 194     for l in o.splitlines():
 195         # Get variable name and value
 196         pkgvar, value = l.split("=")
 197
 198         # If present, strip HOST_ from variable name
 199         if pkgvar.startswith("HOST_"):
 200             pkgvar = pkgvar[5:]
 201
 202         if pkgvar.endswith("_MANIFEST_LICENSE_FILES"):
 203             continue
 204
 205         # Strip _LICENSE_FILES
 206         pkgvar = pkgvar[:-14]
 207
 208         Package.all_license_files.append(pkgvar)
 209
 210
 211 def calculate_stats(packages):
 212     stats = defaultdict(int)
 213     for pkg in packages:
 214         # If packages have multiple infra, take the first one. For the
 215         # vast majority of packages, the target and host infra are the
 216         # same. There are very few packages that use a different infra
 217         # for the host and target variants.
 218         if len(pkg.infras) > 0:
 219             infra = pkg.infras[0][1]
 220             stats["infra-%s" % infra] += 1
 221         else:
 222             stats["infra-unknown"] += 1
 223         if pkg.has_license:
 224             stats["license"] += 1
 225         else:
 226             stats["no-license"] += 1
 227         if pkg.has_license_files:
 228             stats["license-files"] += 1
 229         else:
 230             stats["no-license-files"] += 1
 231         if pkg.has_hash:
 232             stats["hash"] += 1
 233         else:
 234             stats["no-hash"] += 1
 235         stats["patches"] += pkg.patch_count
 236     return stats
 237
 238
 239 html_header = """
 240 <head>
 241 <script src=\"https://www.kryogenix.org/code/browser/sorttable/sorttable.js\"></script>
 242 <style type=\"text/css\">
 243 table {
 244   width: 100%;
 245 }
 246 td {
 247   border: 1px solid black;
 248 }
 249 td.centered {
 250   text-align: center;
 251 }
 252 td.wrong {
 253   background: #ff9a69;
 254 }
 255 td.correct {
 256   background: #d2ffc4;
 257 }
 258 td.nopatches {
 259   background: #d2ffc4;
 260 }
 261 td.somepatches {
 262   background: #ffd870;
 263 }
 264 td.lotsofpatches {
 265   background: #ff9a69;
 266 }
 267 </style>
 268 <title>Statistics of Buildroot packages</title>
 269 </head>
 270
 271 <a href=\"#results\">Results</a><br/>
 272
 273 <p id=\"sortable_hint\"></p>
 274 """
 275
 276
 277 html_footer = """
 278 </body>
 279 <script>
 280 if (typeof sorttable === \"object\") {
 281   document.getElementById(\"sortable_hint\").innerHTML =
 282   \"hint: the table can be sorted by clicking the column headers\"
 283 }
 284 </script>
 285 </html>
 286 """
 287
 288
 289 def infra_str(infra_list):
 290     if not infra_list:
 291         return "Unknown"
 292     elif len(infra_list) == 1:
 293         return "<b>%s</b><br/>%s" % (infra_list[0][1], infra_list[0][0])
 294     elif infra_list[0][1] == infra_list[1][1]:
 295         return "<b>%s</b><br/>%s + %s" % \
 296             (infra_list[0][1], infra_list[0][0], infra_list[1][0])
 297     else:
 298         return "<b>%s</b> (%s)<br/><b>%s</b> (%s)" % \
 299             (infra_list[0][1], infra_list[0][0],
 300              infra_list[1][1], infra_list[1][0])
 301
 302
 303 def boolean_str(b):
 304     if b:
 305         return "Yes"
 306     else:
 307         return "No"
 308
 309
 310 def dump_html_pkg(f, pkg):
 311     f.write(" <tr>\n")
 312     f.write("  <td>%s</td>\n" % pkg.path[2:])
 313
 314     # Patch count
 315     td_class = ["centered"]
 316     if pkg.patch_count == 0:
 317         td_class.append("nopatches")
 318     elif pkg.patch_count < 5:
 319         td_class.append("somepatches")
 320     else:
 321         td_class.append("lotsofpatches")
 322     f.write("  <td class=\"%s\">%s</td>\n" %
 323             (" ".join(td_class), str(pkg.patch_count)))
 324
 325     # Infrastructure
 326     infra = infra_str(pkg.infras)
 327     td_class = ["centered"]
 328     if infra == "Unknown":
 329         td_class.append("wrong")
 330     else:
 331         td_class.append("correct")
 332     f.write("  <td class=\"%s\">%s</td>\n" %
 333             (" ".join(td_class), infra_str(pkg.infras)))
 334
 335     # License
 336     td_class = ["centered"]
 337     if pkg.has_license:
 338         td_class.append("correct")
 339     else:
 340         td_class.append("wrong")
 341     f.write("  <td class=\"%s\">%s</td>\n" %
 342             (" ".join(td_class), boolean_str(pkg.has_license)))
 343
 344     # License files
 345     td_class = ["centered"]
 346     if pkg.has_license_files:
 347         td_class.append("correct")
 348     else:
 349         td_class.append("wrong")
 350     f.write("  <td class=\"%s\">%s</td>\n" %
 351             (" ".join(td_class), boolean_str(pkg.has_license_files)))
 352
 353     # Hash
 354     td_class = ["centered"]
 355     if pkg.has_hash:
 356         td_class.append("correct")
 357     else:
 358         td_class.append("wrong")
 359     f.write("  <td class=\"%s\">%s</td>\n" %
 360             (" ".join(td_class), boolean_str(pkg.has_hash)))
 361
 362     # Warnings
 363     td_class = ["centered"]
 364     if pkg.warnings == 0:
 365         td_class.append("correct")
 366     else:
 367         td_class.append("wrong")
 368     f.write("  <td class=\"%s\">%d</td>\n" %
 369             (" ".join(td_class), pkg.warnings))
 370
 371     f.write(" </tr>\n")
 372
 373
 374 def dump_html_all_pkgs(f, packages):
 375     f.write("""
 376 <table class=\"sortable\">
 377 <tr>
 378 <td>Package</td>
 379 <td class=\"centered\">Patch count</td>
 380 <td class=\"centered\">Infrastructure</td>
 381 <td class=\"centered\">License</td>
 382 <td class=\"centered\">License files</td>
 383 <td class=\"centered\">Hash file</td>
 384 <td class=\"centered\">Warnings</td>
 385 </tr>
 386 """)
 387     for pkg in sorted(packages):
 388         dump_html_pkg(f, pkg)
 389     f.write("</table>")
 390
 391
 392 def dump_html_stats(f, stats):
 393     f.write("<a id=\"results\"></a>\n")
 394     f.write("<table>\n")
 395     infras = [infra[6:] for infra in stats.keys() if infra.startswith("infra-")]
 396     for infra in infras:
 397         f.write(" <tr><td>Packages using the <i>%s</i> infrastructure</td><td>%s</td></tr>\n" %
 398                 (infra, stats["infra-%s" % infra]))
 399     f.write(" <tr><td>Packages having license information</td><td>%s</td></tr>\n" %
 400             stats["license"])
 401     f.write(" <tr><td>Packages not having license information</td><td>%s</td></tr>\n" %
 402             stats["no-license"])
 403     f.write(" <tr><td>Packages having license files information</td><td>%s</td></tr>\n" %
 404             stats["license-files"])
 405     f.write(" <tr><td>Packages not having license files information</td><td>%s</td></tr>\n" %
 406             stats["no-license-files"])
 407     f.write(" <tr><td>Packages having a hash file</td><td>%s</td></tr>\n" %
 408             stats["hash"])
 409     f.write(" <tr><td>Packages not having a hash file</td><td>%s</td></tr>\n" %
 410             stats["no-hash"])
 411     f.write(" <tr><td>Total number of patches</td><td>%s</td></tr>\n" %
 412             stats["patches"])
 413     f.write("</table>\n")
 414
 415
 416 def dump_gen_info(f):
 417     # Updated on Mon Feb 19 08:12:08 CET 2018, Git commit aa77030b8f5e41f1c53eb1c1ad664b8c814ba032
 418     o = subprocess.check_output(["git", "log", "master", "-n", "1", "--pretty=format:%H"])
 419     git_commit = o.splitlines()[0]
 420     f.write("<p><i>Updated on %s, git commit %s</i></p>\n" %
 421             (str(datetime.datetime.utcnow()), git_commit))
 422
 423
 424 def dump_html(packages, stats, output):
 425     with open(output, 'w') as f:
 426         f.write(html_header)
 427         dump_html_all_pkgs(f, packages)
 428         dump_html_stats(f, stats)
 429         dump_gen_info(f)
 430         f.write(html_footer)
 431
 432
 433 def parse_args():
 434     parser = argparse.ArgumentParser()
 435     parser.add_argument('-o', dest='output', action='store', required=True,
 436                         help='HTML output file')
 437     return parser.parse_args()
 438
 439
 440 def __main__():
 441     args = parse_args()
 442     print "Build package list ..."
 443     packages = get_pkglist()
 444     print "Getting package make info ..."
 445     package_init_make_info()
 446     print "Getting package details ..."
 447     for pkg in packages:
 448         pkg.set_infra()
 449         pkg.set_license()
 450         pkg.set_hash_info()
 451         pkg.set_patch_count()
 452         pkg.set_check_package_warnings()
 453     print "Calculate stats"
 454     stats = calculate_stats(packages)
 455     print "Write HTML"
 456     dump_html(packages, stats, args.output)
 457
 458
 459 __main__()