Context Navigation

addons.py @ ferringb%2540gmail.com-20080624174036-q45fk4fxguj9kjlv

Revision ferringb%2540gmail.com-20080624174036-q45fk4fxguj9kjlv, 20.4 kB (checked in by Brian Harring <ferringb@…>, 7 months ago)
punt trailing whitespace

Line
1	# Copyright: 2006 Marien Zwart <marienz@gentoo.org>
2	# Copyright: 2006 Brian Harring <ferringb@gmail.com>
3	# License: GPL2
4
5
6	"""Addon functionality shared by multiple checkers."""
7
8
9	import optparse
10	from itertools import ifilter, ifilterfalse
11	from snakeoil.lists import iflatten_instance
12
13	from pkgcore_checks import base, util
14
15	from snakeoil import (demandload, currying, containers, mappings, iterables,
16	lists)
17	demandload.demandload(globals(),
18	'os',
19	'errno',
20	'snakeoil:osutils',
21	'pkgcore.restrictions:packages,values',
22	'pkgcore.ebuild:misc,domain,profiles',
23	'snakeoil.fileutils:read_dict',
24	'pkgcore.log:logger',
25	)
26
27
28	class ArchesAddon(base.Addon):
29
30	default_arches = tuple(sorted([
31	"x86", "x86-fbsd", "amd64", "ppc", "ppc-macos", "ppc64",
32	"sparc", "mips", "arm", "hppa", "m68k", "ia64", "s390",
33	"sh", "alpha"]))
34
35	@staticmethod
36	def _record_arches(option, opt_str, value, parser):
37	setattr(parser.values, option.dest, tuple(value.split(",")))
38
39	@classmethod
40	def _disable_arches(cls, option, opt_str, value, parser):
41	s = set(getattr(parser.values, 'arches', cls.default_arches))
42	parser.values.arches = tuple(s.difference(value.split(",")))
43
44	@classmethod
45	def mangle_option_parser(cls, parser):
46	parser.add_option(
47	'-a', '--arches', action='callback', callback=cls._record_arches,
48	type='string', default=cls.default_arches,
49	help="comma seperated list of what arches to run, defaults to %s" %
50	",".join(cls.default_arches))
51	parser.add_option(
52	'--disable-arches', action='callback', callback=cls._disable_arches,
53	type='string',
54	help="comma seperated list of arches to disable from the defaults")
55
56
57	class QueryCacheAddon(base.Template):
58
59	priority = 1
60
61	@staticmethod
62	def mangle_option_parser(parser):
63	group = parser.add_option_group('Query caching')
64	group.add_option(
65	'--reset-caching-per', action='store', type='choice',
66	choices=('version', 'package', 'category'),
67	dest='query_caching_freq', default='package',
68	help='control how often the cache is cleared '
69	'(version, package or category)')
70
71	@staticmethod
72	def check_values(values):
73	values.query_caching_freq = {
74	'version': base.versioned_feed,
75	'package': base.package_feed,
76	'category': base.repository_feed,
77	}[values.query_caching_freq]
78
79	def __init__(self, options):
80	base.Addon.__init__(self, options)
81	self.query_cache = {}
82	self.feed_type = self.options.query_caching_freq
83
84	def feed(self, item, reporter):
85	self.query_cache.clear()
86
87
88	class profile_data(object):
89
90	def __init__(self, profile_name, key, virtuals, provides, vfilter,
91	masked_use, forced_use, lookup_cache, insoluable):
92	self.key = key
93	self.name = profile_name
94	self.virtuals = virtuals
95	self.provides_repo = provides
96	self.masked_use = masked_use
97	self.forced_use = forced_use
98	self.cache = lookup_cache
99	self.insoluable = insoluable
100	self.visible = vfilter.match
101
102	def identify_use(self, pkg, known_flags):
103	# note we're trying to be really careful about not creating
104	# pointless intermediate sets unless required
105	# kindly don't change that in any modifications, it adds up.
106	enabled = known_flags.intersection(self.forced_use.pull_data(pkg))
107	immutable = enabled.union(ifilter(known_flags.__contains__,
108	self.masked_use.pull_data(pkg)))
109	return immutable, enabled
110
111
112	class ProfileAddon(base.Addon):
113
114	@staticmethod
115	def check_values(values):
116	if values.profiles_enabled is None:
117	values.profiles_enabled = []
118	if values.profiles_disabled is None:
119	values.profiles_disabled = []
120	profile_loc = getattr(values, "profile_dir", None)
121
122	if profile_loc is not None:
123	if not os.path.isdir(profile_loc):
124	raise optparse.OptionValueError(
125	"profile-base location %r doesn't exist/isn't a dir" % (
126	profile_loc,))
127	else:
128	# TODO improve this to handle multiple profiles dirs.
129	repo_base = getattr(values.src_repo, 'base', None)
130	if repo_base is None:
131	raise optparse.OptionValueError(
132	'Need a target repo or --overlayed-repo that is a single '
133	'UnconfiguredTree for profile checks')
134	profile_loc = osutils.pjoin(repo_base, "profiles")
135	if not os.path.isdir(profile_loc):
136	raise optparse.OptionValueError(
137	"repo %r lacks a profiles directory" % (values.src_repo,))
138
139	profile_loc = osutils.abspath(profile_loc)
140	values.profile_func = currying.pre_curry(profiles.OnDiskProfile,
141	profile_loc)
142	values.profile_base_dir = profile_loc
143
144	@staticmethod
145	def mangle_option_parser(parser):
146	group = parser.add_option_group('Profiles')
147	group.add_option(
148	"--profile-base", action='store', type='string',
149	dest='profile_dir', default=None,
150	help="filepath to base profiles directory")
151	group.add_option(
152	"--profile-disable-dev", action='store_true',
153	default=False, dest='profile_ignore_dev',
154	help="disable scanning of dev profiles")
155	group.add_option(
156	"--profile-disable-deprecated", action='store_true',
157	default=False, dest='profile_ignore_deprecated',
158	help="disable scanning of deprecated profiles")
159	group.add_option(
160	"--profile-disable-profiles-desc", action='store_false',
161	default=True, dest='profiles_desc_enabled',
162	help="disable loading profiles to scan from profiles.desc, you "
163	"will want to enable profiles manually via --profile-enable")
164	group.add_option(
165	'--profile-enable', action='append', type='string',
166	dest='profiles_enabled', help="specify a profile to scan")
167	group.add_option(
168	'--profile-disable', action='append', type='string',
169	dest='profiles_disabled', help="specify a profile to ignore")
170
171	def __init__(self, options, *args):
172	base.Addon.__init__(self, options)
173
174	def norm_name(name):
175	return '/'.join(y for y in name.split('/') if y)
176
177	disabled = set(norm_name(x) for x in options.profiles_disabled)
178	enabled = set(x for x in
179	(norm_name(y) for y in options.profiles_enabled)
180	if x not in disabled)
181
182	arch_profiles = {}
183	if options.profiles_desc_enabled:
184	d = \
185	util.get_profiles_desc(options.profile_base_dir,
186	ignore_dev=options.profile_ignore_dev)
187
188	for k, v in d.iteritems():
189	l = [x for x in map(norm_name, v)
190	if not x in disabled]
191
192	# wipe any enableds that are here already so we don't
193	# get a profile twice
194	enabled.difference_update(l)
195	if v:
196	arch_profiles[k] = l
197
198	for x in enabled:
199	p = options.profile_func(x)
200	arch = p.arch
201	if arch is None:
202	raise profiles.ProfileError(p.path, 'make.defaults',
203	"profile %s lacks arch settings, unable to use it" % x)
204	arch_profiles.setdefault(p.arch, []).append((x, p))
205
206	for x in options.profiles_enabled:
207	options.profile_func(x)
208
209	self.official_arches = util.get_repo_known_arches(
210	options.profile_base_dir)
211
212	self.desired_arches = getattr(self.options, 'arches', None)
213	if self.desired_arches is None:
214	# copy it to be safe
215	self.desired_arches = set(self.official_arches)
216
217	self.global_insoluable = set()
218	profile_filters = {}
219	self.keywords_filter = {}
220	ignore_deprecated = self.options.profile_ignore_deprecated
221
222	# we hold onto the profiles as we're going, due to the fact
223	# profilenodes are weakly cached; hold onto all for this loop,
224	# avoids a lot of reparsing at the expense of slightly more memory
225	# temporarily.
226	cached_profiles = []
227
228	for k in self.desired_arches:
229	if k.lstrip("~") not in self.desired_arches:
230	continue
231	stable_key = k.lstrip("~")
232	unstable_key = "~"+ stable_key
233	stable_r = packages.PackageRestriction("keywords",
234	values.ContainmentMatch(stable_key))
235	unstable_r = packages.PackageRestriction("keywords",
236	values.ContainmentMatch(stable_key, unstable_key))
237
238	default_masked_use = [(packages.AlwaysTrue, (x,)) for x in
239	self.official_arches if x != stable_key]
240
241	profile_filters.update({stable_key:[], unstable_key:[]})
242	for profile_name in arch_profiles.get(k, []):
243	if not isinstance(profile_name, basestring):
244	profile_name, profile = profile_name
245	else:
246	profile = options.profile_func(profile_name)
247	cached_profiles.append(profile)
248	if ignore_deprecated and profile.deprecated:
249	continue
250
251	mask = domain.generate_masking_restrict(profile.masks)
252	virtuals = profile.make_virtuals_repo(options.search_repo)
253
254	immutable_flags = misc.collapsed_restrict_to_data(
255	default_masked_use,
256	profile.masked_use.iteritems())
257	enabled_flags = misc.collapsed_restrict_to_data(
258	[(packages.AlwaysTrue, (stable_key,))],
259	profile.forced_use.iteritems())
260
261	# used to interlink stable/unstable lookups so that if
262	# unstable says it's not visible, stable doesn't try
263	# if stable says something is visible, unstable doesn't try.
264	stable_cache = set()
265	unstable_insoluable = containers.ProtectedSet(
266	self.global_insoluable)
267
268	# few notes. for filter, ensure keywords is last, on the
269	# offchance a non-metadata based restrict foregos having to
270	# access the metadata.
271	# note that the cache/insoluable are inversly paired;
272	# stable cache is usable for unstable, but not vice versa.
273	# unstable insoluable is usable for stable, but not vice versa
274
275	profile_filters[stable_key].append(profile_data(
276	profile_name, stable_key,
277	virtuals, profile.provides_repo,
278	packages.AndRestriction(mask, stable_r),
279	immutable_flags, enabled_flags, stable_cache,
280	containers.ProtectedSet(unstable_insoluable)))
281
282	profile_filters[unstable_key].append(profile_data(
283	profile_name, unstable_key,
284	virtuals, profile.provides_repo,
285	packages.AndRestriction(mask, unstable_r),
286	immutable_flags, enabled_flags,
287	containers.ProtectedSet(stable_cache),
288	unstable_insoluable))
289
290	self.keywords_filter[stable_key] = stable_r
291	self.keywords_filter[unstable_key] = packages.PackageRestriction(
292	"keywords",
293	values.ContainmentMatch(unstable_key))
294
295	profile_evaluate_dict = {}
296	for key, profile_list in profile_filters.iteritems():
297	similar = profile_evaluate_dict[key] = []
298	for profile in profile_list:
299	for existing in similar:
300	if existing[0].masked_use == profile.masked_use and \
301	existing[0].forced_use == profile.forced_use:
302	existing.append(profile)
303	break
304	else:
305	similar.append([profile])
306
307	self.profile_evaluate_dict = profile_evaluate_dict
308	self.arch_profiles = arch_profiles
309	self.keywords_filter = mappings.OrderedDict(
310	(k, self.keywords_filter[k])
311	for k in sorted(self.keywords_filter))
312	self.profile_filters = profile_filters
313
314	def identify_profiles(self, pkg):
315	# yields groups of profiles; the 'groups' are grouped by the ability to share
316	# the use processing across each of 'em.
317	l = []
318	for key in set(pkg.keywords):
319	profile_grps = self.profile_evaluate_dict.get(key)
320	if profile_grps is None:
321	continue
322	for profiles in profile_grps:
323	l2 = [x for x in profiles if x.visible(pkg)]
324	if not l2:
325	continue
326	l.append(l2)
327	return l
328
329
330	class EvaluateDepSetAddon(base.Template):
331
332	required_addons = (ProfileAddon,)
333	feed_type = base.versioned_feed
334	priority = 1
335
336	def __init__(self, options, profiles):
337	base.Addon.__init__(self, options)
338	self.pkg_evaluate_depsets_cache = {}
339	self.pkg_profiles_cache = {}
340	self.profiles = profiles
341
342	def feed(self, item, reporter):
343	self.pkg_evaluate_depsets_cache.clear()
344	self.pkg_profiles_cache.clear()
345
346	def collapse_evaluate_depset(self, pkg, attr, depset):
347	depset_profiles = self.pkg_evaluate_depsets_cache.get((pkg, attr))
348	if depset_profiles is None:
349	depset_profiles = self.identify_common_depsets(pkg, depset)
350	self.pkg_evaluate_depsets_cache[(pkg, attr)] = depset_profiles
351	return depset_profiles
352
353	def identify_common_depsets(self, pkg, depset):
354	profile_grps = self.pkg_profiles_cache.get(pkg, None)
355	if profile_grps is None:
356	profile_grps = self.profiles.identify_profiles(pkg)
357	self.pkg_profiles_cache[pkg] = profile_grps
358	diuse = depset.known_conditionals
359	collapsed = {}
360	for profiles in profile_grps:
361	immutables, enabled = profiles[0].identify_use(pkg, diuse)
362	collapsed.setdefault((immutables, enabled), []).extend(profiles)
363
364	return [(depset.evaluate_depset(k[1], tristate_filter=k[0]), v)
365	for k,v in collapsed.iteritems()]
366
367
368	class LicenseAddon(base.Addon):
369
370	@staticmethod
371	def mangle_option_parser(parser):
372	parser.add_option(
373	"--license-dir", action='store', type='string',
374	help="filepath to license directory")
375
376	@staticmethod
377	def check_values(values):
378	values.license_dirs = []
379	if values.license_dir is None:
380	for repo_base in values.repo_bases:
381	candidate = osutils.pjoin(repo_base, 'licenses')
382	if os.path.isdir(candidate):
383	values.license_dirs.append(candidate)
384	if not values.license_dirs:
385	raise optparse.OptionValueError(
386	'No license dir detected, pick a target or overlayed repo '
387	'with a license dir or specify one with --license-dir.')
388	else:
389	if not os.path.isdir(values.license_dir):
390	raise optparse.OptionValueError(
391	"--license-dir %r isn't a directory" % values.license_dir)
392	values.license_dirs.append(osutils.abspath(values.license_dir))
393
394	@property
395	def licenses(self):
396	o = getattr(self, "_licenses", None)
397	if o is None:
398	o = frozenset(iflatten_instance(
399	osutils.listdir_files(x) for x in self.options.license_dirs))
400	setattr(self, "_licenses", o)
401	return o
402
403
404	class UnstatedIUSE(base.Result):
405	"""pkg is reliant on conditionals that aren't in IUSE"""
406	__slots__ = ("category", "package", "version", "attr", "flags")
407
408	threshold = base.versioned_feed
409
410	def __init__(self, pkg, attr, flags):
411	base.Result.__init__(self)
412	self._store_cpv(pkg)
413	self.attr, self.flags = attr, tuple(flags)
414
415	@property
416	def short_desc(self):
417	return "attr(%s) uses unstated flags [ %s ]" % \
418	(self.attr, ', '.join(self.flags))
419
420
421	class UseAddon(base.Addon):
422
423	known_results = (UnstatedIUSE,)
424
425	def __init__(self, options, silence_warnings=False):
426	base.Addon.__init__(self, options)
427	known_iuse = set()
428	specific_iuse = []
429	unstated_iuse = set()
430	known_arches = set()
431	arches = set()
432
433	for profile_base in options.repo_bases:
434	try:
435	known_iuse.update(util.get_use_desc(
436	osutils.join(profile_base, 'profiles')))
437	except IOError, ie:
438	if ie.errno != errno.ENOENT:
439	raise
440	try:
441	arches.update(util.get_repo_known_arches(
442	osutils.join(profile_base, 'profiles')))
443	except IOError, ie:
444	if ie.errno != errno.ENOENT:
445	raise
446	try:
447	for restricts_dict in \
448	util.get_use_local_desc(
449	osutils.join(profile_base, 'profiles')).itervalues():
450	specific_iuse.extend(restricts_dict.iteritems())
451
452	except IOError, ie:
453	if ie.errno != errno.ENOENT:
454	raise
455
456	use_expand_base = osutils.join(profile_base, "profiles", "desc")
457	try:
458	for entry in osutils.listdir_files(use_expand_base):
459	try:
460	estr = entry.rsplit(".", 1)[0].lower()+ "_"
461	unstated_iuse.update(estr + usef.strip() for usef in
462	read_dict(osutils.join(use_expand_base, entry),
463	None).iterkeys())
464	except (IOError, OSError), ie:
465	if ie.errno != errno.EISDIR:
466	raise
467	del ie
468	except (OSError, IOError), ie:
469	if ie.errno != errno.ENOENT:
470	raise
471
472	self.specific_iuse = tuple((x[0], tuple(x[1])) for x in specific_iuse)
473	self.collapsed_iuse = misc.non_incremental_collapsed_restrict_to_data(
474	((packages.AlwaysTrue, known_iuse),),
475	((packages.AlwaysTrue, unstated_iuse),),
476	self.specific_iuse)
477	self.global_iuse = frozenset(known_iuse)
478	unstated_iuse.update(arches)
479	self.unstated_iuse = frozenset(unstated_iuse)
480	self.profile_bases = profile_base
481	self.ignore = not (unstated_iuse or known_iuse)
482	if self.ignore and not silence_warnings:
483	logger.warn('disabling use/iuse validity checks since no usable '
484	'use.desc, use.local.desc were found ')
485
486	def allowed_iuse(self, pkg):
487	return self.collapsed_iuse.pull_data(pkg)
488
489	def get_filter(self, attr_name=None):
490	if self.ignore:
491	return self.fake_use_validate
492	if attr_name is not None:
493	return currying.partial(self.use_validate, attr=attr_name)
494	return self.use_validate
495
496	@staticmethod
497	def fake_use_validate(klasses, pkg, seq, reporter, attr=None):
498	return iflatten_instance(seq, klasses)
499
500	def iuse_strip(self, iuse):
501	ret_val = set()
502	for flag in iuse:
503	flag = flag.lstrip("+-")
504	ret_val.add(flag)
505	return ret_val
506
507	def use_validate(self, klasses, pkg, seq, reporter, attr=None):
508	skip_filter = (packages.Conditional,) + klasses
509	unstated = set()
510
511	stated = self.iuse_strip(pkg.iuse)
512	i = iterables.expandable_chain(lists.iflatten_instance(seq,
513	skip_filter))
514	for node in i:
515	if isinstance(node, packages.Conditional):
516	# invert it; get only whats not in pkg.iuse
517	unstated.update(ifilterfalse(stated.__contains__,
518	node.restriction.vals))
519	i.append(lists.iflatten_instance(node.payload, skip_filter))
520	continue
521	yield node
522
523	# the valid_unstated_iuse filters out USE_EXPAND as long as
524	# it's listed in a desc file
525	unstated.difference_update(self.unstated_iuse)
526	# hack, see bugs.gentoo.org 134994.
527	unstated.difference_update(["bootstrap"])
528	if unstated:
529	reporter.add_report(UnstatedIUSE(pkg, attr, unstated))

Note: See TracBrowser for help on using the browser.

Download in other formats:

Original Format