support/scripts/pkg-stats-new

   1 #!/usr/bin/env python
   2
   3 # Copyright (C) 2009 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 2 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  13 # General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program; if not, write to the Free Software
  17 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  18
  19 import argparse
  20 import datetime
  21 import fnmatch
  22 import os
  23 from collections import defaultdict
  24 import re
  25 import subprocess
  26 import sys
  27
  28 INFRA_RE = re.compile("\$\(eval \$\(([a-z-]*)-package\)\)")
  29
  30
  31 class Package:
  32     all_licenses = list()
  33     all_license_files = list()
  34
  35     def __init__(self, name, path):
  36         self.name = name
  37         self.path = path
  38         self.infras = None
  39         self.has_license = False
  40         self.has_license_files = False
  41         self.has_hash = False
  42         self.patch_count = 0
  43         self.warnings = 0
  44
  45     def pkgvar(self):
  46         return self.name.upper().replace("-", "_")
  47
  48     def set_infra(self):
  49         """
  50         Fills in the .infras field
  51         """
  52         self.infras = list()
  53         with open(self.path, 'r') as f:
  54             lines = f.readlines()
  55             for l in lines:
  56                 match = INFRA_RE.match(l)
  57                 if not match:
  58                     continue
  59                 infra = match.group(1)
  60                 if infra.startswith("host-"):
  61                     self.infras.append(("host", infra[5:]))
  62                 else:
  63                     self.infras.append(("target", infra))
  64
  65     def set_license(self):
  66         """
  67         Fills in the .has_license and .has_license_files fields
  68         """
  69         var = self.pkgvar()
  70         if var in self.all_licenses:
  71             self.has_license = True
  72         if var in self.all_license_files:
  73             self.has_license_files = True
  74
  75     def set_hash_info(self):
  76         """
  77         Fills in the .has_hash field
  78         """
  79         hashpath = self.path.replace(".mk", ".hash")
  80         self.has_hash = os.path.exists(hashpath)
  81
  82     def set_patch_count(self):
  83         """
  84         Fills in the .patch_count field
  85         """
  86         self.patch_count = 0
  87         pkgdir = os.path.dirname(self.path)
  88         for subdir, _, _ in os.walk(pkgdir):
  89             self.patch_count += len(fnmatch.filter(os.listdir(subdir), '*.patch'))
  90
  91     def set_check_package_warnings(self):
  92         """
  93         Fills in the .warnings field
  94         """
  95         cmd = ["./utils/check-package"]
  96         pkgdir = os.path.dirname(self.path)
  97         for root, dirs, files in os.walk(pkgdir):
  98             for f in files:
  99                 if f.endswith(".mk") or f.endswith(".hash") or f == "Config.in" or f == "Config.in.host":
 100                     cmd.append(os.path.join(root, f))
 101         o = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[1]
 102         lines = o.splitlines()
 103         for line in lines:
 104             m = re.match("^([0-9]*) warnings generated", line)
 105             if m:
 106                 self.warnings = int(m.group(1))
 107                 return
 108
 109     def __eq__(self, other):
 110         return self.path == other.path
 111
 112     def __lt__(self, other):
 113         return self.path < other.path
 114
 115     def __str__(self):
 116         return "%s (path='%s', license='%s', license_files='%s', hash='%s', patches=%d)" % \
 117             (self.name, self.path, self.has_license, self.has_license_files, self.has_hash, self.patch_count)
 118
 119
 120 def get_pkglist(npackages, package_list):
 121     """
 122     Builds the list of Buildroot packages, returning a list of Package
 123     objects. Only the .name and .path fields of the Package object are
 124     initialized.
 125
 126     npackages: limit to N packages
 127     package_list: limit to those packages in this list
 128     """
 129     WALK_USEFUL_SUBDIRS = ["boot", "linux", "package", "toolchain"]
 130     WALK_EXCLUDES = ["boot/common.mk",
 131                      "linux/linux-ext-.*.mk",
 132                      "package/freescale-imx/freescale-imx.mk",
 133                      "package/gcc/gcc.mk",
 134                      "package/gstreamer/gstreamer.mk",
 135                      "package/gstreamer1/gstreamer1.mk",
 136                      "package/gtk2-themes/gtk2-themes.mk",
 137                      "package/matchbox/matchbox.mk",
 138                      "package/opengl/opengl.mk",
 139                      "package/qt5/qt5.mk",
 140                      "package/x11r7/x11r7.mk",
 141                      "package/doc-asciidoc.mk",
 142                      "package/pkg-.*.mk",
 143                      "package/nvidia-tegra23/nvidia-tegra23.mk",
 144                      "toolchain/toolchain-external/pkg-toolchain-external.mk",
 145                      "toolchain/toolchain-external/toolchain-external.mk",
 146                      "toolchain/toolchain.mk",
 147                      "toolchain/helpers.mk",
 148                      "toolchain/toolchain-wrapper.mk"]
 149     packages = list()
 150     count = 0
 151     for root, dirs, files in os.walk("."):
 152         rootdir = root.split("/")
 153         if len(rootdir) < 2:
 154             continue
 155         if rootdir[1] not in WALK_USEFUL_SUBDIRS:
 156             continue
 157         for f in files:
 158             if not f.endswith(".mk"):
 159                 continue
 160             # Strip ending ".mk"
 161             pkgname = f[:-3]
 162             if package_list and pkgname not in package_list:
 163                 continue
 164             pkgpath = os.path.join(root, f)
 165             skip = False
 166             for exclude in WALK_EXCLUDES:
 167                 # pkgpath[2:] strips the initial './'
 168                 if re.match(exclude, pkgpath[2:]):
 169                     skip = True
 170                     continue
 171             if skip:
 172                 continue
 173             p = Package(pkgname, pkgpath)
 174             packages.append(p)
 175             count += 1
 176             if npackages and count == npackages:
 177                 return packages
 178     return packages
 179
 180
 181 def package_init_make_info():
 182     # Licenses
 183     o = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y",
 184                                  "-s", "printvars", "VARS=%_LICENSE"])
 185     for l in o.splitlines():
 186         # Get variable name and value
 187         pkgvar, value = l.split("=")
 188
 189         # If present, strip HOST_ from variable name
 190         if pkgvar.startswith("HOST_"):
 191             pkgvar = pkgvar[5:]
 192
 193         # Strip _LICENSE
 194         pkgvar = pkgvar[:-8]
 195
 196         # If value is "unknown", no license details available
 197         if value == "unknown":
 198             continue
 199         Package.all_licenses.append(pkgvar)
 200
 201     # License files
 202     o = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y",
 203                                  "-s", "printvars", "VARS=%_LICENSE_FILES"])
 204     for l in o.splitlines():
 205         # Get variable name and value
 206         pkgvar, value = l.split("=")
 207
 208         # If present, strip HOST_ from variable name
 209         if pkgvar.startswith("HOST_"):
 210             pkgvar = pkgvar[5:]
 211
 212         if pkgvar.endswith("_MANIFEST_LICENSE_FILES"):
 213             continue
 214
 215         # Strip _LICENSE_FILES
 216         pkgvar = pkgvar[:-14]
 217
 218         Package.all_license_files.append(pkgvar)
 219
 220
 221 def calculate_stats(packages):
 222     stats = defaultdict(int)
 223     for pkg in packages:
 224         # If packages have multiple infra, take the first one. For the
 225         # vast majority of packages, the target and host infra are the
 226         # same. There are very few packages that use a different infra
 227         # for the host and target variants.
 228         if len(pkg.infras) > 0:
 229             infra = pkg.infras[0][1]
 230             stats["infra-%s" % infra] += 1
 231         else:
 232             stats["infra-unknown"] += 1
 233         if pkg.has_license:
 234             stats["license"] += 1
 235         else:
 236             stats["no-license"] += 1
 237         if pkg.has_license_files:
 238             stats["license-files"] += 1
 239         else:
 240             stats["no-license-files"] += 1
 241         if pkg.has_hash:
 242             stats["hash"] += 1
 243         else:
 244             stats["no-hash"] += 1
 245         stats["patches"] += pkg.patch_count
 246     return stats
 247
 248
 249 html_header = """
 250 <head>
 251 <script src=\"https://www.kryogenix.org/code/browser/sorttable/sorttable.js\"></script>
 252 <style type=\"text/css\">
 253 table {
 254   width: 100%;
 255 }
 256 td {
 257   border: 1px solid black;
 258 }
 259 td.centered {
 260   text-align: center;
 261 }
 262 td.wrong {
 263   background: #ff9a69;
 264 }
 265 td.correct {
 266   background: #d2ffc4;
 267 }
 268 td.nopatches {
 269   background: #d2ffc4;
 270 }
 271 td.somepatches {
 272   background: #ffd870;
 273 }
 274 td.lotsofpatches {
 275   background: #ff9a69;
 276 }
 277 </style>
 278 <title>Statistics of Buildroot packages</title>
 279 </head>
 280
 281 <a href=\"#results\">Results</a><br/>
 282
 283 <p id=\"sortable_hint\"></p>
 284 """
 285
 286
 287 html_footer = """
 288 </body>
 289 <script>
 290 if (typeof sorttable === \"object\") {
 291   document.getElementById(\"sortable_hint\").innerHTML =
 292   \"hint: the table can be sorted by clicking the column headers\"
 293 }
 294 </script>
 295 </html>
 296 """
 297
 298
 299 def infra_str(infra_list):
 300     if not infra_list:
 301         return "Unknown"
 302     elif len(infra_list) == 1:
 303         return "<b>%s</b><br/>%s" % (infra_list[0][1], infra_list[0][0])
 304     elif infra_list[0][1] == infra_list[1][1]:
 305         return "<b>%s</b><br/>%s + %s" % \
 306             (infra_list[0][1], infra_list[0][0], infra_list[1][0])
 307     else:
 308         return "<b>%s</b> (%s)<br/><b>%s</b> (%s)" % \
 309             (infra_list[0][1], infra_list[0][0],
 310              infra_list[1][1], infra_list[1][0])
 311
 312
 313 def boolean_str(b):
 314     if b:
 315         return "Yes"
 316     else:
 317         return "No"
 318
 319
 320 def dump_html_pkg(f, pkg):
 321     f.write(" <tr>\n")
 322     f.write("  <td>%s</td>\n" % pkg.path[2:])
 323
 324     # Patch count
 325     td_class = ["centered"]
 326     if pkg.patch_count == 0:
 327         td_class.append("nopatches")
 328     elif pkg.patch_count < 5:
 329         td_class.append("somepatches")
 330     else:
 331         td_class.append("lotsofpatches")
 332     f.write("  <td class=\"%s\">%s</td>\n" %
 333             (" ".join(td_class), str(pkg.patch_count)))
 334
 335     # Infrastructure
 336     infra = infra_str(pkg.infras)
 337     td_class = ["centered"]
 338     if infra == "Unknown":
 339         td_class.append("wrong")
 340     else:
 341         td_class.append("correct")
 342     f.write("  <td class=\"%s\">%s</td>\n" %
 343             (" ".join(td_class), infra_str(pkg.infras)))
 344
 345     # License
 346     td_class = ["centered"]
 347     if pkg.has_license:
 348         td_class.append("correct")
 349     else:
 350         td_class.append("wrong")
 351     f.write("  <td class=\"%s\">%s</td>\n" %
 352             (" ".join(td_class), boolean_str(pkg.has_license)))
 353
 354     # License files
 355     td_class = ["centered"]
 356     if pkg.has_license_files:
 357         td_class.append("correct")
 358     else:
 359         td_class.append("wrong")
 360     f.write("  <td class=\"%s\">%s</td>\n" %
 361             (" ".join(td_class), boolean_str(pkg.has_license_files)))
 362
 363     # Hash
 364     td_class = ["centered"]
 365     if pkg.has_hash:
 366         td_class.append("correct")
 367     else:
 368         td_class.append("wrong")
 369     f.write("  <td class=\"%s\">%s</td>\n" %
 370             (" ".join(td_class), boolean_str(pkg.has_hash)))
 371
 372     # Warnings
 373     td_class = ["centered"]
 374     if pkg.warnings == 0:
 375         td_class.append("correct")
 376     else:
 377         td_class.append("wrong")
 378     f.write("  <td class=\"%s\">%d</td>\n" %
 379             (" ".join(td_class), pkg.warnings))
 380
 381     f.write(" </tr>\n")
 382
 383
 384 def dump_html_all_pkgs(f, packages):
 385     f.write("""
 386 <table class=\"sortable\">
 387 <tr>
 388 <td>Package</td>
 389 <td class=\"centered\">Patch count</td>
 390 <td class=\"centered\">Infrastructure</td>
 391 <td class=\"centered\">License</td>
 392 <td class=\"centered\">License files</td>
 393 <td class=\"centered\">Hash file</td>
 394 <td class=\"centered\">Warnings</td>
 395 </tr>
 396 """)
 397     for pkg in sorted(packages):
 398         dump_html_pkg(f, pkg)
 399     f.write("</table>")
 400
 401
 402 def dump_html_stats(f, stats):
 403     f.write("<a id=\"results\"></a>\n")
 404     f.write("<table>\n")
 405     infras = [infra[6:] for infra in stats.keys() if infra.startswith("infra-")]
 406     for infra in infras:
 407         f.write(" <tr><td>Packages using the <i>%s</i> infrastructure</td><td>%s</td></tr>\n" %
 408                 (infra, stats["infra-%s" % infra]))
 409     f.write(" <tr><td>Packages having license information</td><td>%s</td></tr>\n" %
 410             stats["license"])
 411     f.write(" <tr><td>Packages not having license information</td><td>%s</td></tr>\n" %
 412             stats["no-license"])
 413     f.write(" <tr><td>Packages having license files information</td><td>%s</td></tr>\n" %
 414             stats["license-files"])
 415     f.write(" <tr><td>Packages not having license files information</td><td>%s</td></tr>\n" %
 416             stats["no-license-files"])
 417     f.write(" <tr><td>Packages having a hash file</td><td>%s</td></tr>\n" %
 418             stats["hash"])
 419     f.write(" <tr><td>Packages not having a hash file</td><td>%s</td></tr>\n" %
 420             stats["no-hash"])
 421     f.write(" <tr><td>Total number of patches</td><td>%s</td></tr>\n" %
 422             stats["patches"])
 423     f.write("</table>\n")
 424
 425
 426 def dump_gen_info(f):
 427     # Updated on Mon Feb 19 08:12:08 CET 2018, Git commit aa77030b8f5e41f1c53eb1c1ad664b8c814ba032
 428     o = subprocess.check_output(["git", "log", "master", "-n", "1", "--pretty=format:%H"])
 429     git_commit = o.splitlines()[0]
 430     f.write("<p><i>Updated on %s, git commit %s</i></p>\n" %
 431             (str(datetime.datetime.utcnow()), git_commit))
 432
 433
 434 def dump_html(packages, stats, output):
 435     with open(output, 'w') as f:
 436         f.write(html_header)
 437         dump_html_all_pkgs(f, packages)
 438         dump_html_stats(f, stats)
 439         dump_gen_info(f)
 440         f.write(html_footer)
 441
 442
 443 def parse_args():
 444     parser = argparse.ArgumentParser()
 445     parser.add_argument('-o', dest='output', action='store', required=True,
 446                         help='HTML output file')
 447     parser.add_argument('-n', dest='npackages', type=int, action='store',
 448                         help='Number of packages')
 449     parser.add_argument('-p', dest='packages', action='store',
 450                         help='List of packages (comma separated)')
 451     return parser.parse_args()
 452
 453
 454 def __main__():
 455     args = parse_args()
 456     if args.npackages and args.packages:
 457         print "ERROR: -n and -p are mutually exclusive"
 458         sys.exit(1)
 459     if args.packages:
 460         package_list = args.packages.split(",")
 461     else:
 462         package_list = None
 463     print "Build package list ..."
 464     packages = get_pkglist(args.npackages, package_list)
 465     print "Getting package make info ..."
 466     package_init_make_info()
 467     print "Getting package details ..."
 468     for pkg in packages:
 469         pkg.set_infra()
 470         pkg.set_license()
 471         pkg.set_hash_info()
 472         pkg.set_patch_count()
 473         pkg.set_check_package_warnings()
 474     print "Calculate stats"
 475     stats = calculate_stats(packages)
 476     print "Write HTML"
 477     dump_html(packages, stats, args.output)
 478
 479
 480 __main__()