root/pkgcore-checks/pkgcore_checks/pcheck.py @ ferringb%2540gmail.com-20081019013033-usrpmp9dd8t1amsf

Revision ferringb%2540gmail.com-20081019013033-usrpmp9dd8t1amsf, 21.5 kB (checked in by Brian Harring <ferringb@…>, 3 months ago)

silence 'running %i tests' exempting debugging
Line 
1# Copyright: 2006 Brian Harring <ferringb@gmail.com>
2# Copyright: 2006 Marien Zwart <marienz@gentoo.org>
3# License: GPL2
4
5
6"""Commandline frontend (for use with L{pkgcore.util.commandline.main}."""
7
8
9from pkgcore.util import commandline, parserestrict
10from pkgcore.plugin import get_plugins, get_plugin
11from pkgcore_checks import plugins
12from pkgcore_checks import plugins, base, __version__, feeds
13from snakeoil import lists, demandload
14from snakeoil.formatters import decorate_forced_wrapping
15
16demandload.demandload(globals(),
17    'optparse',
18    'textwrap',
19    'os',
20    'logging',
21    'snakeoil:osutils',
22    'pkgcore.restrictions:packages',
23    'pkgcore.restrictions.values:StrExactMatch',
24    'pkgcore.repository:multiplex',
25    'pkgcore.ebuild:repository',
26    'pkgcore_checks:errors',
27)
28
29
30def repo_callback(option, opt_str, value, parser):
31    try:
32        repo = parser.values.config.repo[value]
33    except KeyError:
34        raise optparse.OptionValueError(
35            'repo %r is not a known repo (known repos: %s)' % (
36                value, ', '.join(repr(n) for n in parser.values.config.repo)))
37    if not isinstance(repo, repository.UnconfiguredTree):
38        raise optparse.OptionValueError(
39            'repo %r is not a pkgcore.ebuild.repository.UnconfiguredTree '
40            'instance; must specify a raw ebuild repo, not type %r: %r' % (
41                value, repo.__class__, repo))
42    setattr(parser.values, option.dest, repo)
43
44
45class OptionParser(commandline.OptionParser):
46
47    """Option parser that is automagically extended by the checks.
48
49    Some comments on the resulting values object:
50
51    - target_repo is passed in as first argument and used as source for
52      packages to check.
53    - src_repo is specified with -r or defaults to target_repo. It is used
54      to get the profiles directory and other non-package repository data.
55    - repo_bases are the path(s) to selected repo(s).
56    - search_repo is a multiplex of target_repo and src_repo if they are
57      different or just target_repo if they are the same. This is used for
58      things like visibility checks (it is passed to the checkers in "start").
59    """
60
61    def __init__(self, **kwargs):
62        commandline.OptionParser.__init__(
63            self, version='pkgcore-checks %s' % (__version__,),
64            description="pkgcore based ebuild QA checks",
65            usage="usage: %prog [options] [atom1...atom2]",
66            **kwargs)
67
68        # These are all set in check_values based on other options, so have
69        # no default set through add_option.
70        self.set_default('repo_bases', [])
71        self.set_default('guessed_target_repo', False)
72        self.set_default('guessed_suite', False)
73        self.set_default('default_suite', False)
74
75        group = self.add_option_group('Check selection')
76        group.add_option(
77            "-c", action="append", type="string", dest="checks_to_run",
78            help="limit checks to those matching this regex, or package/class "
79            "matching; may be specified multiple times")
80        group.set_conflict_handler("resolve")
81        group.add_option("-d",
82            "--disable", action="append", type="string",
83            dest="checks_to_disable", help="specific checks to disable: "
84            "may be specified multiple times")
85        group.set_conflict_handler("error")
86        group.add_option(
87            '--checkset', action='callback', type='string',
88            callback=commandline.config_callback,
89            callback_args=('pcheck_checkset', 'checkset'),
90            help='Pick a preconfigured set of checks to run.')
91
92        self.add_option(
93            '--repo', '-r', action='callback', type='string',
94            callback=repo_callback, dest='target_repo',
95            help='Set the target repo')
96        self.add_option(
97            '--suite', '-s', action='callback', type='string',
98            callback=commandline.config_callback,
99            callback_args=('pcheck_suite', 'suite'),
100            help='Specify the configuration suite to use')
101        self.add_option(
102            "--list-checks", action="store_true", default=False,
103            help="print what checks are available to run and exit")
104        self.add_option(
105            '--reporter', type='string', action='store', default=None,
106            help="Use a non-default reporter (defined in pkgcore's config).")
107        self.add_option(
108            '--list-reporters', action='store_true', default=False,
109            help="print known reporters")
110
111        overlay = self.add_option_group('Overlay')
112        overlay.add_option(
113            '--overlayed-repo', '-o', action='callback', type='string',
114            callback=repo_callback, dest='src_repo',
115            help="if the target repository is an overlay, specify the "
116            "repository name to pull profiles/license from")
117
118        all_addons = set()
119        def add_addon(addon):
120            if addon not in all_addons:
121                all_addons.add(addon)
122                for dep in addon.required_addons:
123                    add_addon(dep)
124        for check in get_plugins('check', plugins):
125            add_addon(check)
126        for addon in all_addons:
127            addon.mangle_option_parser(self)
128
129    def check_values(self, values, args):
130        values, args = commandline.OptionParser.check_values(
131            self, values, args)
132        # XXX hack...
133        values.checks = sorted(get_plugins('check', plugins))
134        if values.list_checks or values.list_reporters:
135            if values.list_reporters == values.list_checks:
136                raise optparse.OptionValueError("--list-checks and "
137                    "--list-reporters are mutually exclusive options- "
138                    "one or the other.")
139            return values, ()
140        cwd = None
141        if values.suite is None:
142            # No suite explicitly specified. Use the repo to guess the suite.
143            if values.target_repo is None:
144                # Not specified either. Try to find a repo our cwd is in.
145                cwd = os.getcwd()
146                # The use of a dict here is a hack to deal with one
147                # repo having multiple names in the configuration.
148                candidates = {}
149                for name, suite in values.config.pcheck_suite.iteritems():
150                    repo = suite.target_repo
151                    if repo is None:
152                        continue
153                    repo_base = getattr(repo, 'base', None)
154                    if repo_base is not None and cwd.startswith(repo_base):
155                        candidates[repo] = name
156                if len(candidates) == 1:
157                    values.guessed_suite = True
158                    values.target_repo = tuple(candidates)[0]
159            if values.target_repo is not None:
160                # We have a repo, now find a suite matching it.
161                candidates = list(
162                    suite for suite in values.config.pcheck_suite.itervalues()
163                    if suite.target_repo is values.target_repo)
164                if len(candidates) == 1:
165                    values.guessed_suite = True
166                    values.suite = candidates[0]
167            if values.suite is None:
168                # If we have multiple candidates or no candidates we
169                # fall back to the default suite.
170                values.suite = values.config.get_default('pcheck_suite')
171                values.default_suite = values.suite is not None
172        if values.suite is not None:
173            # We have a suite. Lift defaults from it for values that
174            # were not set explicitly:
175            if values.checkset is None:
176                values.checkset = values.suite.checkset
177            if values.src_repo is None:
178                values.src_repo = values.suite.src_repo
179            # If we were called with no atoms we want to force
180            # cwd-based detection.
181            if values.target_repo is None:
182                if args:
183                    values.target_repo = values.suite.target_repo
184                elif values.suite.target_repo is not None:
185                    # No atoms were passed in, so we want to guess
186                    # what to scan based on cwd below. That only makes
187                    # sense if we are inside the target repo. We still
188                    # want to pick the suite's target repo if we are
189                    # inside it, in case there is more than one repo
190                    # definition with a base that contains our dir.
191                    if cwd is None:
192                        cwd = os.getcwd()
193                    repo_base = getattr(values.suite.target_repo, 'base', None)
194                    if repo_base is not None and cwd.startswith(repo_base):
195                        values.target_repo = values.suite.target_repo
196        if values.target_repo is None:
197            # We have no target repo (not explicitly passed, not from
198            # a suite, not from an earlier guess at the target_repo).
199            # Try to guess one from cwd:
200            if cwd is None:
201                cwd = os.getcwd()
202            candidates = {}
203            for name, repo in values.config.repo.iteritems():
204                repo_base = getattr(repo, 'base', None)
205                if repo_base is not None and cwd.startswith(repo_base):
206                    candidates[repo] = name
207            if not candidates:
208                self.error(
209                    'No target repo specified on commandline or suite and '
210                    'current directory is not inside a known repo.')
211            elif len(candidates) > 1:
212                self.error(
213                    'Found multiple matches when guessing repo based on '
214                    'current directory (%s). Specify a repo on the '
215                    'commandline or suite or remove some repos from your '
216                    'configuration.' % (
217                        ', '.join(str(repo) for repo in candidates),))
218            values.target_repo = tuple(candidates)[0]
219
220        if values.reporter is None:
221            values.reporter = values.config.get_default(
222                'pcheck_reporter_factory')
223            if values.reporter is None:
224                values.reporter = get_plugin('reporter', plugins)
225            if values.reporter is None:
226                self.error('no config defined reporter found, nor any default '
227                    'plugin based reporters')
228        else:
229            func = values.config.pcheck_reporter_factory.get(values.reporter)
230            if func is None:
231                func = list(base.Whitelist([values.reporter]).filter(
232                    get_plugins('reporter', plugins)))
233                if not func:
234                    self.error("no reporter matches %r\n"
235                        "please see --list-reporter for a list of "
236                        "valid reporters" % values.reporter)
237                elif len(func) > 1:
238                    self.error("--reporter %r matched multiple reporters, "
239                        "must match one. %r" %
240                            (values.reporter,
241                                tuple(sorted("%s.%s" %
242                                    (x.__module__, x.__name__)
243                                    for x in func))
244                            )
245                    )
246                func = func[0]
247            values.reporter = func
248        if values.src_repo is None:
249            values.src_repo = values.target_repo
250            values.search_repo = values.target_repo
251        else:
252            values.search_repo = multiplex.tree(values.target_repo,
253                                                values.src_repo)
254
255        # TODO improve this to deal with a multiplex repo.
256        for repo in set((values.src_repo, values.target_repo)):
257            if isinstance(repo, repository.UnconfiguredTree):
258                values.repo_bases.append(osutils.abspath(repo.base))
259
260        if args:
261            values.limiters = lists.stable_unique(map(
262                    parserestrict.parse_match, args))
263        else:
264            repo_base = getattr(values.target_repo, 'base', None)
265            if not repo_base:
266                self.error(
267                    'Either specify a target repo that is not multi-tree or '
268                    'one or more extended atoms to scan '
269                    '("*" for the entire repo).')
270            cwd = osutils.abspath(os.getcwd())
271            repo_base = osutils.abspath(repo_base)
272            if not cwd.startswith(repo_base):
273                self.error(
274                    'Working dir (%s) is not inside target repo (%s). Fix '
275                    'that or specify one or more extended atoms to scan.' % (
276                        cwd, repo_base))
277            bits = list(p for p in cwd[len(repo_base):].split(os.sep) if p)
278            if not bits:
279                values.limiters = [packages.AlwaysTrue]
280            elif len(bits) == 1:
281                values.limiters = [packages.PackageRestriction(
282                        'category', StrExactMatch(bits[0]))]
283            else:
284                values.limiters = [packages.AndRestriction(
285                        packages.PackageRestriction(
286                            'category', StrExactMatch(bits[0])),
287                        packages.PackageRestriction(
288                            'package', StrExactMatch(bits[1])))]
289
290        if values.checkset is None:
291            values.checkset = values.config.get_default('pcheck_checkset')
292        if values.checkset is not None:
293            values.checks = list(values.checkset.filter(values.checks))
294
295        if values.checks_to_run:
296            whitelist = base.Whitelist(values.checks_to_run)
297            values.checks = list(whitelist.filter(values.checks))
298
299        if values.checks_to_disable:
300            blacklist = base.Blacklist(values.checks_to_disable)
301            values.checks = list(blacklist.filter(values.checks))
302
303        if not values.checks:
304            self.error('No active checks')
305
306        values.addons = set()
307        def add_addon(addon):
308            if addon not in values.addons:
309                values.addons.add(addon)
310                for dep in addon.required_addons:
311                    add_addon(dep)
312        for check in values.checks:
313            add_addon(check)
314        try:
315            for addon in values.addons:
316                addon.check_values(values)
317        except optparse.OptionValueError, e:
318            if values.debug:
319                raise
320            self.error(str(e))
321
322        return values, ()
323
324
325def dump_docstring(out, obj, prefix=None):
326    if prefix is not None:
327        out.first_prefix.append(prefix)
328        out.later_prefix.append(prefix)
329    try:
330        if obj.__doc__ is None:
331            out.write("no documentation")
332            return
333
334        # Docstrings start with an unindented line. Everything
335        # else is consistently indented.
336        lines = obj.__doc__.split('\n')
337        firstline = lines[0].strip()
338        # Some docstrings actually start on the second line.
339        if firstline:
340            out.write(firstline)
341        if len(lines) > 1:
342            for line in textwrap.dedent('\n'.join(lines[1:])).split('\n'):
343                if line:
344                    out.write(line)
345    finally:
346        if prefix is not None:
347            out.first_prefix.pop()
348            out.later_prefix.pop()
349
350@decorate_forced_wrapping()
351def display_checks(out, checks):
352    d = {}
353    for x in checks:
354        d.setdefault(x.__module__, []).append(x)
355
356    if not d:
357        out.write(out.fg('red'), "No Documentation")
358        out.write()
359        return
360
361    for module_name in sorted(d):
362        out.write(out.bold, "%s:" % module_name)
363        l = d[module_name]
364        l.sort(key=lambda x:x.__name__)
365
366        try:
367            out.first_prefix.append('  ')
368            out.later_prefix.append('  ')
369            for check in l:
370                out.write("%s:" % check.__name__)
371                dump_docstring(out, check, prefix='  ')
372            out.write()
373        finally:
374            out.first_prefix.pop()
375            out.later_prefix.pop()
376
377
378@decorate_forced_wrapping()
379def display_reporters(out, config, config_reporters, plugin_reporters):
380    out.write("known reporters:")
381    out.write()
382    if config_reporters:
383        out.write("configured reporters:")
384        out.first_prefix.append(' ')
385        out.later_prefix.append(' ')
386        try:
387            # sorting here is random
388            for reporter in sorted(config_reporters, key=lambda x:x.__name__):
389                key = config.get_section_name(reporter)
390                if not key:
391                    continue
392                out.write(out.bold, key)
393                dump_docstring(out, reporter, prefix=' ')
394                out.write()
395        finally:
396            out.first_prefix.pop()
397            out.later_prefix.pop()
398
399    if plugin_reporters:
400        if config_reporters:
401            out.write()
402        out.write("plugin reporters:")
403        out.first_prefix.append(' ')
404        out.later_prefix.append(' ')
405        try:
406            for reporter in sorted(plugin_reporters, key=lambda x:x.__name__):
407                out.write(out.bold, reporter.__name__)
408                dump_docstring(out, reporter, prefix=' ')
409                out.write()
410        finally:
411            out.first_prefix.pop()
412            out.later_prefix.pop()
413
414    if not plugin_reporters and not config_reporters:
415        out.write(out.fg("red"), "Warning", out.fg(""),
416            " no reporters detected; pcheck won't "
417            "run correctly without a reporter to use!")
418        out.write()
419
420def main(options, out, err):
421    """Do stuff."""
422
423    if options.list_checks:
424        display_checks(out, options.checks)
425        return 0
426
427    if options.list_reporters:
428        display_reporters(out, options.config,
429            options.config.pcheck_reporter_factory.values(),
430            list(get_plugins('reporter', plugins)))
431        return 0
432
433    if not options.repo_bases:
434        err.write(
435            'Warning: could not determine repository base for profiles. '
436            'Some checks will not work. Either specify a plain target repo '
437            '(not combined trees) or specify a PORTDIR repo '
438            'with --overlayed-repo.', wrap=True)
439        err.write()
440
441    if options.guessed_suite:
442        if options.default_suite:
443            err.write('Tried to guess a suite to use but got multiple matches')
444            err.write('and fell back to the default.')
445        else:
446            err.write('using suite guessed from working directory')
447
448    if options.guessed_target_repo:
449        err.write('using repository guessed from working directory')
450
451    try:
452        reporter = options.reporter(out)
453    except errors.ReporterInitError, e:
454        err.write(err.fg('red'), err.bold, '!!! ', err.reset,
455                  'Error initializing reporter: ', e)
456        return 1
457
458    addons_map = {}
459    def init_addon(klass):
460        res = addons_map.get(klass)
461        if res is not None:
462            return res
463        deps = list(init_addon(dep) for dep in klass.required_addons)
464        try:
465            res = addons_map[klass] = klass(options, *deps)
466        except KeyboardInterrupt:
467            raise
468        except Exception:
469            err.write('instantiating %s' % (klass,))
470            raise
471        return res
472
473    for addon in options.addons:
474        # Ignore the return value, we just need to populate addons_map.
475        init_addon(addon)
476
477    if options.debug:
478        err.write('target repo: ', repr(options.target_repo))
479        err.write('source repo: ', repr(options.src_repo))
480        err.write('base dirs: ', repr(options.repo_bases))
481        for filterer in options.limiters:
482            err.write('limiter: ', repr(filterer))
483        debug = logging.debug
484    else:
485        debug = None
486
487    transforms = list(get_plugins('transform', plugins))
488    # XXX this is pretty horrible.
489    sinks = list(addon for addon in addons_map.itervalues()
490                 if getattr(addon, 'feed_type', False))
491
492    reporter.start()
493
494    for filterer in options.limiters:
495        sources = [feeds.RestrictedRepoSource(options.target_repo, filterer)]
496        bad_sinks, pipes = base.plug(sinks, transforms, sources, debug)
497        if bad_sinks:
498            # We want to report the ones that would work if this was a
499            # full repo scan separately from the ones that are
500            # actually missing transforms.
501            bad_sinks = set(bad_sinks)
502            full_scope = feeds.RestrictedRepoSource(options.target_repo,
503                                                    packages.AlwaysTrue)
504            really_bad, ignored = base.plug(sinks, transforms, [full_scope])
505            really_bad = set(really_bad)
506            assert bad_sinks >= really_bad, \
507                '%r unreachable with no limiters but reachable with?' % (
508                really_bad - bad_sinks,)
509            out_of_scope = bad_sinks - really_bad
510            for sink in really_bad:
511                err.error(
512                    'sink %s could not be connected (missing transforms?)' % (
513                        sink,))
514            for sink in bad_sinks - really_bad:
515                err.warn('not running %s (not a full repo scan)' % (
516                        sink.__class__.__name__,))
517        if not pipes:
518            out.write(out.fg('red'), ' * ', out.reset, 'No checks!')
519        else:
520            if options.debug:
521                err.write('Running %i tests' % (len(sinks) - len(bad_sinks),))
522            for source, pipe in pipes:
523                pipe.start()
524                reporter.start_check(list(base.collect_checks_classes(pipe)),
525                    filterer)
526                for thing in source.feed():
527                    pipe.feed(thing, reporter)
528                pipe.finish(reporter)
529                reporter.end_check()
530
531    reporter.finish()
532
533    # flush stdout first; if they're directing it all to a file, this makes
534    # results not get the final message shoved in midway
535    out.stream.flush()
536    return 0
Note: See TracBrowser for help on using the browser.