Context Navigation

addons.py @ ferringb%2540gmail.com-20070211040449-7gbt4jyorbf6fb6f

Revision ferringb%2540gmail.com-20070211040449-7gbt4jyorbf6fb6f, 19.8 KB (checked in by Brian Harring <ferringb@…>, 2 years ago)
shift generating the set of known licenses to the addon

Line
1	# Copyright: 2006 Marien Zwart <marienz@gentoo.org>
2	# Copyright: 2006 Brian Harring <ferringb@gmail.com>
3	# License: GPL2
4
5
6	"""Addon functionality shared by multiple checkers."""
7
8
9	import optparse
10	from itertools import ifilter, ifilterfalse
11	from pkgcore.util.lists import iflatten_instance
12
13	from pkgcore_checks import base, util
14
15	from pkgcore.util import (
16	demandload, currying, containers, mappings, iterables, lists)
17	demandload.demandload(
18	globals(),
19	'os '
20	'errno '
21	'pkgcore.util:osutils '
22	'pkgcore.restrictions:packages,values '
23	'pkgcore.ebuild:misc,domain,profiles '
24	'pkgcore.util.file:read_dict '
25	'pkgcore.log:logger '
26	)
27
28
29	class ArchesAddon(base.Addon):
30
31	default_arches = tuple(sorted([
32	"x86", "x86-fbsd", "amd64", "ppc", "ppc-macos", "ppc64",
33	"sparc", "mips", "arm", "hppa", "m68k", "ia64", "s390",
34	"sh", "alpha"]))
35
36	@staticmethod
37	def _record_arches(option, opt_str, value, parser):
38	setattr(parser.values, option.dest, tuple(value.split(",")))
39
40	@classmethod
41	def mangle_option_parser(cls, parser):
42	parser.add_option(
43	'-a', '--arches', action='callback', callback=cls._record_arches,
44	type='string', default=cls.default_arches,
45	help="comma seperated list of what arches to run, defaults to %s" %
46	",".join(cls.default_arches))
47
48
49	class QueryCacheAddon(base.Template):
50
51	priority = 1
52
53	@staticmethod
54	def mangle_option_parser(parser):
55	group = parser.add_option_group('Query caching')
56	group.add_option(
57	'--reset-caching-per', action='store', type='choice',
58	choices=('version', 'package', 'category'),
59	dest='query_caching_freq', default='package',
60	help='control how often the cache is cleared '
61	'(version, package or category)')
62
63	@staticmethod
64	def check_values(values):
65	values.query_caching_freq = {
66	'version': base.versioned_feed,
67	'package': base.package_feed,
68	'category': base.repository_feed,
69	}[values.query_caching_freq]
70
71	def __init__(self, options):
72	base.Addon.__init__(self, options)
73	self.query_cache = {}
74	self.feed_type = self.options.query_caching_freq
75
76	def feed(self, item, reporter):
77	self.query_cache.clear()
78
79
80	class profile_data(object):
81
82	def __init__(self, profile_name, key, virtuals, provides, vfilter,
83	masked_use, forced_use, lookup_cache, insoluable):
84	self.key = key
85	self.name = profile_name
86	self.virtuals = virtuals
87	self.provides_repo = provides
88	self.masked_use = masked_use
89	self.forced_use = forced_use
90	self.cache = lookup_cache
91	self.insoluable = insoluable
92	self.visible = vfilter.match
93
94	def identify_use(self, pkg, known_flags):
95	# note we're trying to be really careful about not creating
96	# pointless intermediate sets unless required
97	# kindly don't change that in any modifications, it adds up.
98	enabled = known_flags.intersection(self.forced_use.pull_data(pkg))
99	immutable = enabled.union(ifilter(known_flags.__contains__,
100	self.masked_use.pull_data(pkg)))
101	return immutable, enabled
102
103
104	class ProfileAddon(base.Addon):
105
106	@staticmethod
107	def check_values(values):
108	if values.profiles_enabled is None:
109	values.profiles_enabled = []
110	if values.profiles_disabled is None:
111	values.profiles_disabled = []
112	profile_loc = getattr(values, "profile_dir", None)
113
114	if profile_loc is not None:
115	if not os.path.isdir(profile_loc):
116	raise optparse.OptionValueError(
117	"profile-base location %r doesn't exist/isn't a dir" % (
118	profile_loc,))
119	else:
120	# TODO improve this to handle multiple profiles dirs.
121	repo_base = getattr(values.src_repo, 'base', None)
122	if repo_base is None:
123	raise optparse.OptionValueError(
124	'Need a target repo or --overlayed-repo that is a single '
125	'UnconfiguredTree for profile checks')
126	profile_loc = osutils.pjoin(repo_base, "profiles")
127	if not os.path.isdir(profile_loc):
128	raise optparse.OptionValueError(
129	"repo %r lacks a profiles directory" % (values.src_repo,))
130
131	profile_loc = osutils.abspath(profile_loc)
132	values.profile_func = currying.pre_curry(profiles.OnDiskProfile,
133	profile_loc)
134	values.profile_base_dir = profile_loc
135
136	@staticmethod
137	def mangle_option_parser(parser):
138	group = parser.add_option_group('Profiles')
139	group.add_option(
140	"--profile-base", action='store', type='string',
141	dest='profile_dir', default=None,
142	help="filepath to base profiles directory")
143	group.add_option(
144	"--profile-disable-dev", action='store_true',
145	default=False, dest='profile_ignore_dev',
146	help="disable scanning of dev profiles")
147	group.add_option(
148	"--profile-disable-deprecated", action='store_true',
149	default=False, dest='profile_ignore_deprecated',
150	help="disable scanning of deprecated profiles")
151	group.add_option(
152	"--profile-disable-profiles-desc", action='store_false',
153	default=True, dest='profiles_desc_enabled',
154	help="disable loading profiles to scan from profiles.desc, you "
155	"will want to enable profiles manually via --profile-enable")
156	group.add_option(
157	'--profile-enable', action='append', type='string',
158	dest='profiles_enabled', help="specify a profile to scan")
159	group.add_option(
160	'--profile-disable', action='append', type='string',
161	dest='profiles_disabled', help="specify a profile to ignore")
162
163	def __init__(self, options, *args):
164	base.Addon.__init__(self, options)
165
166	def norm_name(name):
167	return '/'.join(y for y in name.split('/') if y)
168
169	disabled = set(norm_name(x) for x in options.profiles_disabled)
170	enabled = set(x for x in
171	(norm_name(y) for y in options.profiles_enabled)
172	if x not in disabled)
173
174	arch_profiles = {}
175	if options.profiles_desc_enabled:
176	d = \
177	util.get_profiles_desc(options.profile_base_dir,
178	ignore_dev=options.profile_ignore_dev)
179
180	for k, v in d.iteritems():
181	l = [x for x in map(norm_name, v)
182	if not x in disabled]
183
184	# wipe any enableds that are here already so we don't
185	# get a profile twice
186	enabled.difference_update(l)
187	if v:
188	arch_profiles[k] = l
189
190	for x in enabled:
191	p = options.profile_func(x)
192	arch = p.arch
193	if arch is None:
194	raise profiles.ProfileError(p.path, 'make.defaults',
195	"profile %s lacks arch settings, unable to use it" % x)
196	arch_profiles.setdefault(p.arch, []).append((x, p))
197
198	for x in options.profiles_enabled:
199	options.profile_func(x)
200
201	self.official_arches = util.get_repo_known_arches(
202	options.profile_base_dir)
203
204	self.desired_arches = getattr(self.options, 'arches', None)
205	if self.desired_arches is None:
206	# copy it to be safe
207	self.desired_arches = set(self.official_arches)
208
209	self.global_insoluable = set()
210	profile_filters = {}
211	self.keywords_filter = {}
212	ignore_deprecated = self.options.profile_ignore_deprecated
213
214	for k in self.desired_arches:
215	if k.lstrip("~") not in self.desired_arches:
216	continue
217	stable_key = k.lstrip("~")
218	unstable_key = "~"+ stable_key
219	stable_r = packages.PackageRestriction("keywords",
220	values.ContainmentMatch(stable_key))
221	unstable_r = packages.PackageRestriction("keywords",
222	values.ContainmentMatch(stable_key, unstable_key))
223
224	default_masked_use = [(packages.AlwaysTrue, (x,)) for x in
225	self.official_arches if x != stable_key]
226
227	profile_filters.update({stable_key:[], unstable_key:[]})
228	for profile_name in arch_profiles.get(k, []):
229	if not isinstance(profile_name, basestring):
230	profile_name, profile = profile_name
231	else:
232	profile = options.profile_func(profile_name)
233	if ignore_deprecated and profile.deprecated:
234	continue
235
236	mask = domain.generate_masking_restrict(profile.masks)
237	virtuals = profile.make_virtuals_repo(options.search_repo)
238
239	immutable_flags = misc.collapsed_restrict_to_data(
240	default_masked_use,
241	profile.masked_use.iteritems())
242	enabled_flags = misc.collapsed_restrict_to_data(
243	[(packages.AlwaysTrue, (stable_key,))],
244	profile.forced_use.iteritems())
245
246	# used to interlink stable/unstable lookups so that if
247	# unstable says it's not visible, stable doesn't try
248	# if stable says something is visible, unstable doesn't try.
249	stable_cache = set()
250	unstable_insoluable = containers.ProtectedSet(
251	self.global_insoluable)
252
253	# few notes. for filter, ensure keywords is last, on the
254	# offchance a non-metadata based restrict foregos having to
255	# access the metadata.
256	# note that the cache/insoluable are inversly paired;
257	# stable cache is usable for unstable, but not vice versa.
258	# unstable insoluable is usable for stable, but not vice versa
259
260	profile_filters[stable_key].append(profile_data(
261	profile_name, stable_key,
262	virtuals, profile.provides_repo,
263	packages.AndRestriction(mask, stable_r),
264	immutable_flags, enabled_flags, stable_cache,
265	containers.ProtectedSet(unstable_insoluable)))
266
267	profile_filters[unstable_key].append(profile_data(
268	profile_name, unstable_key,
269	virtuals, profile.provides_repo,
270	packages.AndRestriction(mask, unstable_r),
271	immutable_flags, enabled_flags,
272	containers.ProtectedSet(stable_cache),
273	unstable_insoluable))
274
275	self.keywords_filter[stable_key] = stable_r
276	self.keywords_filter[unstable_key] = packages.PackageRestriction(
277	"keywords",
278	values.ContainmentMatch(unstable_key))
279
280	profile_evaluate_dict = {}
281	for key, profile_list in profile_filters.iteritems():
282	similar = profile_evaluate_dict[key] = []
283	for profile in profile_list:
284	for existing in similar:
285	if existing[0].masked_use == profile.masked_use and \
286	existing[0].forced_use == profile.forced_use:
287	existing.append(profile)
288	break
289	else:
290	similar.append([profile])
291
292	self.profile_evaluate_dict = profile_evaluate_dict
293	self.arch_profiles = arch_profiles
294	self.keywords_filter = mappings.OrderedDict(
295	(k, self.keywords_filter[k])
296	for k in sorted(self.keywords_filter))
297	self.profile_filters = profile_filters
298
299	def identify_profiles(self, pkg):
300	# yields groups of profiles; the 'groups' are grouped by the ability to share
301	# the use processing across each of 'em.
302	l = []
303	for key in set(pkg.keywords):
304	profile_grps = self.profile_evaluate_dict.get(key)
305	if profile_grps is None:
306	continue
307	for profiles in profile_grps:
308	l2 = [x for x in profiles if x.visible(pkg)]
309	if not l2:
310	continue
311	l.append(l2)
312	return l
313
314
315	class EvaluateDepSetAddon(base.Template):
316
317	required_addons = (ProfileAddon,)
318	feed_type = base.versioned_feed
319	priority = 1
320
321	def __init__(self, options, profiles):
322	base.Addon.__init__(self, options)
323	self.pkg_evaluate_depsets_cache = {}
324	self.pkg_profiles_cache = {}
325	self.profiles = profiles
326
327	def feed(self, item, reporter):
328	self.pkg_evaluate_depsets_cache.clear()
329	self.pkg_profiles_cache.clear()
330
331	def collapse_evaluate_depset(self, pkg, attr, depset):
332	depset_profiles = self.pkg_evaluate_depsets_cache.get((pkg, attr))
333	if depset_profiles is None:
334	depset_profiles = self.identify_common_depsets(pkg, depset)
335	self.pkg_evaluate_depsets_cache[(pkg, attr)] = depset_profiles
336	return depset_profiles
337
338	def identify_common_depsets(self, pkg, depset):
339	profile_grps = self.pkg_profiles_cache.get(pkg, None)
340	if profile_grps is None:
341	profile_grps = self.profiles.identify_profiles(pkg)
342	self.pkg_profiles_cache[pkg] = profile_grps
343	diuse = depset.known_conditionals
344	collapsed = {}
345	for profiles in profile_grps:
346	immutables, enabled = profiles[0].identify_use(pkg, diuse)
347	collapsed.setdefault((immutables, enabled), []).extend(profiles)
348
349	return [(depset.evaluate_depset(k[1], tristate_filter=k[0]), v)
350	for k,v in collapsed.iteritems()]
351
352
353	class LicenseAddon(base.Addon):
354
355	@staticmethod
356	def mangle_option_parser(parser):
357	parser.add_option(
358	"--license-dir", action='store', type='string',
359	help="filepath to license directory")
360
361	@staticmethod
362	def check_values(values):
363	values.license_dirs = []
364	if values.license_dir is None:
365	for repo_base in values.repo_bases:
366	candidate = osutils.pjoin(repo_base, 'licenses')
367	if os.path.isdir(candidate):
368	values.license_dirs.append(candidate)
369	if not values.license_dirs:
370	raise optparse.OptionValueError(
371	'No license dir detected, pick a target or overlayed repo '
372	'with a license dir or specify one with --license-dir.')
373	else:
374	if not os.path.isdir(values.license_dir):
375	raise optparse.OptionValueError(
376	"--license-dir %r isn't a directory" % values.license_dir)
377	values.license_dirs.append(osutils.abspath(values.license_dir))
378
379	@property
380	def licenses(self):
381	o = getattr(self, "_licenses", None)
382	if o is None:
383	o = frozenset(iflatten_instance(
384	osutils.listdir_files(x) for x in self.options.license_dirs))
385	setattr(self, "_licenses", o)
386	return o
387
388
389	class UnstatedIUSE(base.Result):
390	"""pkg is reliant on conditionals that aren't in IUSE"""
391	__slots__ = ("category", "package", "version", "attr", "flags")
392
393	threshold = base.versioned_feed
394
395	def __init__(self, pkg, attr, flags):
396	base.Result.__init__(self)
397	self._store_cpv(pkg)
398	self.attr, self.flags = attr, tuple(flags)
399
400	@property
401	def short_desc(self):
402	return "attr(%s) uses unstated flags [ %s ]" % \
403	(self.attr, ', '.join(self.flags))
404
405
406	class UseAddon(base.Addon):
407
408	known_results = (UnstatedIUSE,)
409
410	def __init__(self, options, silence_warnings=False):
411	base.Addon.__init__(self, options)
412	known_iuse = set()
413	specific_iuse = []
414	unstated_iuse = set()
415	known_arches = set()
416	unstated_iuse = set()
417
418	arches = set()
419	for profile_base in options.repo_bases:
420	try:
421	known_iuse.update(util.get_use_desc(
422	osutils.join(profile_base, 'profiles')))
423	except IOError, ie:
424	if ie.errno != errno.ENOENT:
425	raise
426	try:
427	arches.update(util.get_repo_known_arches(
428	osutils.join(profile_base, 'profiles')))
429	except IOError, ie:
430	if ie.errno != errno.ENOENT:
431	raise
432	try:
433	for restricts_dict in \
434	util.get_use_local_desc(
435	osutils.join(profile_base, 'profiles')).itervalues():
436	specific_iuse.extend(restricts_dict.iteritems())
437
438	except IOError, ie:
439	if ie.errno != errno.ENOENT:
440	raise
441
442	use_expand_base = osutils.join(profile_base, "profiles", "desc")
443	try:
444	for entry in osutils.listdir_files(use_expand_base):
445	try:
446	estr = entry.rsplit(".", 1)[0].lower()+ "_"
447	unstated_iuse.update(estr + usef.strip() for usef in
448	read_dict(osutils.join(use_expand_base, entry),
449	None).iterkeys())
450	except (IOError, OSError), ie:
451	if ie.errno != errno.EISDIR:
452	raise
453	del ie
454	except (OSError, IOError), ie:
455	if ie.errno != errno.ENOENT:
456	raise
457
458	self.specific_iuse = tuple((x[0], tuple(x[1])) for x in specific_iuse)
459	self.collapsed_iuse = misc.non_incremental_collapsed_restrict_to_data(
460	((packages.AlwaysTrue, known_iuse),),
461	((packages.AlwaysTrue, unstated_iuse),),
462	self.specific_iuse)
463	self.global_iuse = frozenset(known_iuse)
464	unstated_iuse.update(arches)
465	self.unstated_iuse = frozenset(unstated_iuse)
466	self.profile_bases = profile_base
467	self.ignore = not (unstated_iuse or known_iuse)
468	if self.ignore and not silence_warnings:
469	logger.warn('disabling use/iuse validity checks since no usable '
470	'use.desc, use.local.desc were found ')
471
472	def allowed_iuse(self, pkg):
473	return self.collapsed_iuse.pull_data(pkg)
474
475	def get_filter(self):
476	if self.ignore:
477	return self.fake_use_validate
478	return self.use_validate
479
480	@staticmethod
481	def fake_use_validate(klasses, pkg, seq, reporter):
482	return iflatten_instance(seq, klasses)
483
484	def use_validate(self, klasses, pkg, seq, reporter):
485	skip_filter = (packages.Conditional,) + klasses
486	unstated = set()
487
488	stated = pkg.iuse
489	i = iterables.expandable_chain(lists.iflatten_instance(seq,
490	skip_filter))
491	for node in i:
492	if isinstance(node, packages.Conditional):
493	# invert it; get only whats not in pkg.iuse
494	unstated.update(ifilterfalse(stated.__contains__,
495	node.restriction.vals))
496	i.append(lists.iflatten_instance(node.payload, skip_filter))
497	continue
498	yield node
499
500	# the valid_unstated_iuse filters out USE_EXPAND as long as
501	# it's listed in a desc file
502	unstated.difference_update(self.unstated_iuse)
503	# hack, see bugs.gentoo.org 134994.
504	unstated.difference_update(["bootstrap"])
505	if unstated:
506	if seq == pkg.depends:
507	attr_name = "depends"
508	elif seq == pkg.rdepends:
509	attr_name = "rdepends"
510	elif seq == pkg.post_rdepends:
511	attr_name = "post_rdepends"
512	elif seq == pkg.provides:
513	attr_name = "provide"
514	reporter.add_report(UnstatedIUSE(pkg, attr_name,
515	unstated))

Note: See TracBrowser for help on using the browser.

Download in other formats:

Original Format