From cd034f40e6f1e6d8bcaac924083d7029a2429159 Mon Sep 17 00:00:00 2001 From: Alexander Bersenev Date: Sun, 21 Aug 2011 17:23:29 +0000 Subject: portage integration patch is added --- integration_with_portage.patch | 899 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 899 insertions(+) create mode 100644 integration_with_portage.patch diff --git a/integration_with_portage.patch b/integration_with_portage.patch new file mode 100644 index 0000000..a74beed --- /dev/null +++ b/integration_with_portage.patch @@ -0,0 +1,899 @@ +diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py +index 0144cfc..1c423a3 100644 +--- a/pym/_emerge/EbuildBuild.py ++++ b/pym/_emerge/EbuildBuild.py +@@ -9,6 +9,8 @@ from _emerge.CompositeTask import CompositeTask + from _emerge.EbuildMerge import EbuildMerge + from _emerge.EbuildFetchonly import EbuildFetchonly + from _emerge.EbuildBuildDir import EbuildBuildDir ++from _emerge.EventsAnalyser import EventsAnalyser, FilterProcGenerator ++from _emerge.EventsLogger import EventsLogger + from _emerge.MiscFunctionsProcess import MiscFunctionsProcess + from portage.util import writemsg + import portage +@@ -21,7 +23,7 @@ from portage.package.ebuild._spawn_nofetch import spawn_nofetch + class EbuildBuild(CompositeTask): + + __slots__ = ("args_set", "config_pool", "find_blockers", +- "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", ++ "ldpath_mtimes", "logger", "logserver", "opts", "pkg", "pkg_count", + "prefetcher", "settings", "world_atom") + \ + ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") + +@@ -244,8 +246,54 @@ class EbuildBuild(CompositeTask): + + build = EbuildExecuter(background=self.background, pkg=pkg, + scheduler=scheduler, settings=settings) ++ ++ build.addStartListener(self._build_start) ++ build.addExitListener(self._build_stop) ++ + self._start_task(build, self._build_exit) + ++ def _build_start(self,phase): ++ if "depcheck" in self.settings["FEATURES"] or \ ++ "depcheckstrict" in self.settings["FEATURES"]: ++ # Lets start a log listening server ++ temp_path=self.settings.get("T",self.settings["PORTAGE_TMPDIR"]) ++ ++ if "depcheckstrict" not in self.settings["FEATURES"]: ++ # use default filter_proc ++ self.logserver=EventsLogger(socket_dir=temp_path) ++ else: ++ portage.util.writemsg("Getting list of allowed files..." + \ ++ "This may take some time\n") ++ filter_gen=FilterProcGenerator(self.pkg.cpv, self.settings) ++ filter_proc=filter_gen.get_filter_proc() ++ self.logserver=EventsLogger(socket_dir=temp_path, ++ filter_proc=filter_proc) ++ ++ self.logserver.start() ++ ++ # Copy socket path to LOG_SOCKET environment variable ++ env=self.settings.configdict["pkg"] ++ env['LOG_SOCKET'] = self.logserver.socket_name ++ ++ #import pdb; pdb.set_trace() ++ ++ def _build_stop(self,phase): ++ if "depcheck" in self.settings["FEATURES"] or \ ++ "depcheckstrict" in self.settings["FEATURES"]: ++ # Delete LOG_SOCKET from environment ++ env=self.settings.configdict["pkg"] ++ if 'LOG_SOCKET' in env: ++ del env['LOG_SOCKET'] ++ ++ events=self.logserver.stop() ++ self.logserver=None ++ analyser=EventsAnalyser(self.pkg.cpv, events, self.settings) ++ analyser.display() # show the analyse ++ ++ #import pdb; pdb.set_trace() ++ ++ ++ + def _fetch_failed(self): + # We only call the pkg_nofetch phase if either RESTRICT=fetch + # is set or the package has explicitly overridden the default +diff --git a/pym/_emerge/EbuildPhase.py b/pym/_emerge/EbuildPhase.py +index f53570a..82c165d 100644 +--- a/pym/_emerge/EbuildPhase.py ++++ b/pym/_emerge/EbuildPhase.py +@@ -33,7 +33,8 @@ class EbuildPhase(CompositeTask): + ("_ebuild_lock",) + + # FEATURES displayed prior to setup phase +- _features_display = ("ccache", "distcc", "distcc-pump", "fakeroot", ++ _features_display = ("ccache", "depcheck", "depcheckstrict" "distcc", ++ "distcc-pump", "fakeroot", + "installsources", "keeptemp", "keepwork", "nostrip", + "preserve-libs", "sandbox", "selinux", "sesandbox", + "splitdebug", "suidctl", "test", "userpriv", +diff --git a/pym/_emerge/EventsAnalyser.py b/pym/_emerge/EventsAnalyser.py +new file mode 100644 +index 0000000..65ece7b +--- /dev/null ++++ b/pym/_emerge/EventsAnalyser.py +@@ -0,0 +1,511 @@ ++# Distributed under the terms of the GNU General Public License v2 ++ ++import portage ++from portage.dbapi._expand_new_virt import expand_new_virt ++from portage import os ++ ++import subprocess ++import re ++ ++class PortageUtils: ++ """ class for accessing the portage api """ ++ def __init__(self, settings): ++ """ test """ ++ self.settings=settings ++ self.vartree=portage.vartree(settings=settings) ++ self.vardbapi=portage.vardbapi(settings=settings, vartree=self.vartree) ++ self.portdbapi=portage.portdbapi(mysettings=settings) ++ self.metadata_keys = [k for k in portage.auxdbkeys if not k.startswith("UNUSED_")] ++ self.use=self.settings["USE"] ++ ++ def get_best_visible_pkg(self,pkg): ++ """ ++ Gets best candidate on installing. Returns empty string if no found ++ ++ :param pkg: package name ++ ++ """ ++ try: ++ return self.portdbapi.xmatch("bestmatch-visible", pkg) ++ except: ++ return '' ++ ++ # non-recursive dependency getter ++ def get_dep(self,pkg,dep_type=["RDEPEND","DEPEND"]): ++ """ ++ Gets current dependencies of a package. Looks in portage db ++ ++ :param pkg: name of package ++ :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or ++ ["RDEPEND", "DEPEND"] ++ :returns: **set** of packages names ++ """ ++ ret=set() ++ ++ pkg = self.get_best_visible_pkg(pkg) ++ if not pkg: ++ return ret ++ ++ # we found the best visible match in common tree ++ ++ ++ metadata = dict(zip(self.metadata_keys, ++ self.portdbapi.aux_get(pkg, self.metadata_keys))) ++ dep_str = " ".join(metadata[k] for k in dep_type) ++ ++ # the IUSE default are very important for us ++ iuse_defaults=[ ++ u[1:] for u in metadata.get("IUSE",'').split() if u.startswith("+")] ++ ++ use=self.use.split() ++ ++ for u in iuse_defaults: ++ if u not in use: ++ use.append(u) ++ ++ success, atoms = portage.dep_check(dep_str, None, self.settings, ++ myuse=use, myroot=self.settings["ROOT"], ++ trees={self.settings["ROOT"]:{"vartree":self.vartree, "porttree": self.vartree}}) ++ if not success: ++ return ret ++ ++ for atom in atoms: ++ atomname = self.vartree.dep_bestmatch(atom) ++ ++ if not atomname: ++ continue ++ ++ for unvirt_pkg in expand_new_virt(self.vardbapi,'='+atomname): ++ for pkg in self.vartree.dep_match(unvirt_pkg): ++ ret.add(pkg) ++ ++ return ret ++ ++ # recursive dependency getter ++ def get_deps(self,pkg,dep_type=["RDEPEND","DEPEND"]): ++ """ ++ Gets current dependencies of a package on any depth ++ All dependencies **must** be installed ++ ++ :param pkg: name of package ++ :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or ++ ["RDEPEND", "DEPEND"] ++ :returns: **set** of packages names ++ """ ++ ret=set() ++ ++ ++ # get porttree dependencies on the first package ++ ++ pkg = self.portdbapi.xmatch("bestmatch-visible", pkg) ++ if not pkg: ++ return ret ++ ++ known_packages=set() ++ unknown_packages=self.get_dep(pkg,dep_type) ++ ret=ret.union(unknown_packages) ++ ++ while unknown_packages: ++ p=unknown_packages.pop() ++ if p in known_packages: ++ continue ++ known_packages.add(p) ++ ++ metadata = dict(zip(self.metadata_keys, self.vardbapi.aux_get(p, self.metadata_keys))) ++ ++ dep_str = " ".join(metadata[k] for k in dep_type) ++ ++ # the IUSE default are very important for us ++ iuse_defaults=[ ++ u[1:] for u in metadata.get("IUSE",'').split() if u.startswith("+")] ++ ++ use=self.use.split() ++ ++ for u in iuse_defaults: ++ if u not in use: ++ use.append(u) ++ ++ success, atoms = portage.dep_check(dep_str, None, self.settings, ++ myuse=use, myroot=self.settings["ROOT"], ++ trees={self.settings["ROOT"]:{"vartree":self.vartree,"porttree": self.vartree}}) ++ ++ if not success: ++ continue ++ ++ for atom in atoms: ++ atomname = self.vartree.dep_bestmatch(atom) ++ if not atomname: ++ continue ++ ++ for unvirt_pkg in expand_new_virt(self.vardbapi,'='+atomname): ++ for pkg in self.vartree.dep_match(unvirt_pkg): ++ ret.add(pkg) ++ unknown_packages.add(pkg) ++ return ret ++ ++ def get_deps_for_package_building(self, pkg): ++ """ ++ returns buildtime dependencies of current package and ++ all runtime dependencies of that buildtime dependencies ++ """ ++ buildtime_deps=self.get_dep(pkg, ["DEPEND"]) ++ runtime_deps=set() ++ for dep in buildtime_deps: ++ runtime_deps=runtime_deps.union(self.get_deps(dep,["RDEPEND"])) ++ ++ ret=buildtime_deps.union(runtime_deps) ++ return ret ++ ++ def get_system_packages_list(self): ++ """ ++ returns all packages from system set. They are always implicit dependencies ++ ++ :returns: **list** of package names ++ """ ++ ret=[] ++ for atom in self.settings.packages: ++ for pre_pkg in self.vartree.dep_match(atom): ++ for unvirt_pkg in expand_new_virt(self.vardbapi,'='+pre_pkg): ++ for pkg in self.vartree.dep_match(unvirt_pkg): ++ ret.append(pkg) ++ return ret ++ ++ ++class GentoolkitUtils: ++ """ ++ Interface with qfile and qlist utils. They are much faster than ++ internals. ++ """ ++ ++ def getpackagesbyfiles(files): ++ """ ++ :param files: list of filenames ++ :returns: **dictionary** file->package, if file doesn't belong to any ++ package it not returned as key of this dictionary ++ """ ++ ret={} ++ listtocheck=[] ++ for f in files: ++ if os.path.isdir(f): ++ ret[f]="directory" ++ else: ++ listtocheck.append(f) ++ ++ try: ++ proc=subprocess.Popen(['qfile']+['--nocolor','--exact','','--from','-'], ++ stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE, ++ bufsize=4096) ++ ++ out,err=proc.communicate("\n".join(listtocheck).encode("utf8")) ++ ++ lines=out.decode("utf8").split("\n") ++ #print lines ++ line_re=re.compile(r"^([^ ]+)\s+\(([^)]+)\)$") ++ for line in lines: ++ if len(line)==0: ++ continue ++ match=line_re.match(line) ++ if match: ++ ret[match.group(2)]=match.group(1) ++ else: ++ portage.util.writemsg("Util qfile returned unparsable string: %s\n" % line) ++ ++ except OSError as e: ++ portage.util.writemsg("Error while launching qfile: %s\n" % e) ++ ++ ++ return ret ++ ++ def getfilesbypackages(packagenames): ++ """ ++ ++ :param packagename: name of package ++ :returns: **list** of files in package with name *packagename* ++ """ ++ ret=[] ++ try: ++ proc=subprocess.Popen(['qlist']+['--nocolor',"--obj"]+packagenames, ++ stdout=subprocess.PIPE,stderr=subprocess.PIPE, ++ bufsize=4096) ++ ++ out,err=proc.communicate() ++ ++ ret=out.decode("utf8").split("\n") ++ if ret==['']: ++ ret=[] ++ except OSError as e: ++ portage.util.writemsg("Error while launching qfile: %s\n" % e) ++ ++ return ret ++ ++ def get_all_packages_files(): ++ """ ++ Memory-hungry operation ++ ++ :returns: **set** of all files that belongs to package ++ """ ++ ret=[] ++ try: ++ proc=subprocess.Popen(['qlist']+['--all',"--obj"], ++ stdout=subprocess.PIPE,stderr=subprocess.PIPE, ++ bufsize=4096) ++ ++ out,err=proc.communicate() ++ ++ ret=out.decode("utf8").split("\n") ++ except OSError as e: ++ portage.util.writemsg("Error while launching qfile: %s\n" % e) ++ ++ return set(ret) ++ ++class FilterProcGenerator: ++ def __init__(self, pkgname, settings): ++ portageutils=PortageUtils(settings=settings) ++ ++ deps_all=portageutils.get_deps_for_package_building(pkgname) ++ deps_portage=portageutils.get_dep('portage',["RDEPEND"]) ++ ++ system_packages=portageutils.get_system_packages_list() ++ ++ allfiles=GentoolkitUtils.get_all_packages_files() ++ portage.util.writemsg("All files list recieved, waiting for " \ ++ "a list of allowed files\n") ++ ++ ++ allowedpkgs=system_packages+list(deps_portage)+list(deps_all) ++ ++ allowedfiles=GentoolkitUtils.getfilesbypackages(allowedpkgs) ++ #for pkg in allowedpkgs: ++ # allowedfiles+=GentoolkitUtils.getfilesbypackage(pkg) ++ ++ #import pdb; pdb.set_trace() ++ ++ # manually add all python interpreters to this list ++ allowedfiles+=GentoolkitUtils.getfilesbypackages(['python']) ++ allowedfiles=set(allowedfiles) ++ ++ deniedfiles=allfiles-allowedfiles ++ ++ def filter_proc(eventname,filename,stage): ++ if filename in deniedfiles: ++ return False ++ return True ++ ++ self.filter_proc=filter_proc ++ def get_filter_proc(self): ++ return self.filter_proc ++ ++class EventsAnalyser: ++ def __init__(self, pkgname, events, settings): ++ self.pkgname=pkgname ++ self.events=events ++ self.settings=settings ++ self.portageutils=PortageUtils(settings=settings) ++ ++ self.deps_all=self.portageutils.get_deps_for_package_building(pkgname) ++ self.deps_direct=self.portageutils.get_dep(pkgname,["DEPEND"]) ++ self.deps_portage=self.portageutils.get_dep('portage',["RDEPEND"]) ++ ++ self.system_packages=self.portageutils.get_system_packages_list() ++ # All analyse work is here ++ ++ # get unique filenames ++ filenames=set() ++ for stage in events: ++ succ_events=set(events[stage][0]) ++ fail_events=set(events[stage][1]) ++ filenames=filenames.union(succ_events) ++ filenames=filenames.union(fail_events) ++ filenames=list(filenames) ++ ++ file_to_package=GentoolkitUtils.getpackagesbyfiles(filenames) ++ # This part is completly unreadable. ++ # It converting one complex struct(returned by getfsevents) to another complex ++ # struct which good for generating output. ++ # ++ # Old struct is also used during output ++ ++ packagesinfo={} ++ ++ for stage in sorted(events): ++ succ_events=events[stage][0] ++ fail_events=events[stage][1] ++ ++ for filename in succ_events: ++ if filename in file_to_package: ++ package=file_to_package[filename] ++ else: ++ package="unknown" ++ ++ if not package in packagesinfo: ++ packagesinfo[package]={} ++ stageinfo=packagesinfo[package] ++ if not stage in stageinfo: ++ stageinfo[stage]={} ++ ++ filesinfo=stageinfo[stage] ++ if not filename in filesinfo: ++ filesinfo[filename]={"found":[],"notfound":[]} ++ filesinfo[filename]["found"]=succ_events[filename] ++ ++ for filename in fail_events: ++ if filename in file_to_package: ++ package=file_to_package[filename] ++ else: ++ package="unknown" ++ if not package in packagesinfo: ++ packagesinfo[package]={} ++ stageinfo=packagesinfo[package] ++ if not stage in stageinfo: ++ stageinfo[stage]={} ++ ++ filesinfo=stageinfo[stage] ++ if not filename in filesinfo: ++ filesinfo[filename]={"found":[],"notfound":[]} ++ filesinfo[filename]["notfound"]=fail_events[filename] ++ self.packagesinfo=packagesinfo ++ ++ def display(self): ++ portage.util.writemsg( ++ portage.output.colorize( ++ "WARN", "\nFile access report for %s:\n" % self.pkgname)) ++ ++ stagesorder={"clean":1,"setup":2,"unpack":3,"prepare":4,"configure":5,"compile":6,"test":7, ++ "install":8,"preinst":9,"postinst":10,"prerm":11,"postrm":12,"unknown":13} ++ packagesinfo=self.packagesinfo ++ # print information grouped by package ++ for package in sorted(packagesinfo): ++ # not showing special directory package ++ if package=="directory": ++ continue ++ ++ if package=="unknown": ++ continue ++ ++ ++ is_pkg_in_dep=package in self.deps_all ++ is_pkg_in_portage_dep=package in self.deps_portage ++ is_pkg_in_system=package in self.system_packages ++ is_pkg_python="dev-lang/python" in package ++ ++ stages=[] ++ for stage in sorted(packagesinfo[package].keys(), key=stagesorder.get): ++ if stage!="unknown": ++ stages.append(stage) ++ ++ if len(stages)==0: ++ continue ++ ++ filenames={} ++ for stage in stages: ++ for filename in packagesinfo[package][stage]: ++ if len(packagesinfo[package][stage][filename]["found"])!=0: ++ was_readed,was_writed=packagesinfo[package][stage][filename]["found"] ++ if not filename in filenames: ++ filenames[filename]=['ok',was_readed,was_writed] ++ else: ++ status, old_was_readed, old_was_writed=filenames[filename] ++ filenames[filename]=[ ++ 'ok',old_was_readed | was_readed, old_was_writed | was_writed ++ ] ++ if len(packagesinfo[package][stage][filename]["notfound"])!=0: ++ was_notfound,was_blocked=packagesinfo[package][stage][filename]["notfound"] ++ if not filename in filenames: ++ filenames[filename]=['err',was_notfound,was_blocked] ++ else: ++ status, old_was_notfound, old_was_blocked=filenames[filename] ++ filenames[filename]=[ ++ 'err',old_was_notfound | was_notfound, old_was_blocked | was_blocked ++ ] ++ ++ ++ if is_pkg_in_dep: ++ portage.util.writemsg("[OK]") ++ elif is_pkg_in_system: ++ portage.util.writemsg("[SYSTEM]") ++ elif is_pkg_in_portage_dep: ++ portage.util.writemsg("[PORTAGE DEP]") ++ elif is_pkg_python: ++ portage.util.writemsg("[INTERPRETER]") ++ elif not self.is_package_useful(package,stages,filenames.keys()): ++ portage.util.writemsg("[LIKELY OK]") ++ else: ++ portage.util.writemsg(portage.output.colorize("BAD", "[NOT IN DEPS]")) ++ # show information about accessed files ++ ++ portage.util.writemsg(" %-40s: %s\n" % (package,stages)) ++ ++ # this is here for readability ++ action={ ++ ('ok',False,False):"accessed", ++ ('ok',True,False):"readed", ++ ('ok',False,True):"writed", ++ ('ok',True,True):"readed and writed", ++ ('err',False,False):"other error", ++ ('err',True,False):"not found", ++ ('err',False,True):"blocked", ++ ('err',True,True):"not found and blocked" ++ } ++ ++ filescounter=0 ++ ++ for filename in filenames: ++ event_info=tuple(filenames[filename]) ++ portage.util.writemsg(" %-56s %-21s\n" % (filename,action[event_info])) ++ filescounter+=1 ++ if filescounter>10: ++ portage.util.writemsg(" ... and %d more ...\n" % (len(filenames)-10)) ++ break ++ # ... and one more check. Making sure that direct build time ++ # dependencies were accessed ++ #import pdb; pdb.set_trace() ++ not_accessed_deps=set(self.deps_direct)-set(self.packagesinfo.keys()) ++ if not_accessed_deps: ++ portage.util.writemsg(portage.output.colorize("WARN", "!!! ")) ++ portage.util.writemsg("Warning! Some build time dependencies " + \ ++ "of packages were not accessed: " + \ ++ " ".join(not_accessed_deps) + "\n") ++ ++ def is_package_useful(self,pkg,stages,files): ++ """ some basic heuristics here to cut part of packages """ ++ ++ excluded_paths=set( ++ ['/etc/sandbox.d/'] ++ ) ++ ++ excluded_packages=set( ++ # autodep shows these two packages every time ++ ['net-zope/zope-fixers', 'net-zope/zope-interface'] ++ ) ++ ++ ++ def is_pkg_excluded(p): ++ for pkg in excluded_packages: ++ if p.startswith(pkg): # if package is excluded ++ return True ++ return False ++ ++ ++ def is_file_excluded(f): ++ for path in excluded_paths: ++ if f.startswith(path): # if path is excluded ++ return True ++ return False ++ ++ ++ if is_pkg_excluded(pkg): ++ return False ++ ++ for f in files: ++ if is_file_excluded(f): ++ continue ++ ++ # test 1: package is not useful if all files are *.desktop or *.xml or *.m4 ++ if not (f.endswith(".desktop") or f.endswith(".xml") or f.endswith(".m4") or f.endswith(".pc")): ++ break ++ else: ++ return False # we get here if cycle ends not with break ++ ++ return True ++ ++ +\ No newline at end of file +diff --git a/pym/_emerge/EventsLogger.py b/pym/_emerge/EventsLogger.py +new file mode 100644 +index 0000000..68b3c67 +--- /dev/null ++++ b/pym/_emerge/EventsLogger.py +@@ -0,0 +1,180 @@ ++# Distributed under the terms of the GNU General Public License v2 ++ ++import io ++import sys ++import stat ++import socket ++import select ++import tempfile ++ ++import threading ++ ++from portage import os ++ ++class EventsLogger(threading.Thread): ++ def default_filter(eventname, filename, stage): ++ return True ++ ++ def __init__(self, socket_dir="/tmp/", filter_proc=default_filter): ++ threading.Thread.__init__(self) # init the Thread ++ ++ self.alive=False ++ ++ self.main_thread=threading.currentThread() ++ ++ self.socket_dir=socket_dir ++ self.filter_proc=filter_proc ++ ++ self.socket_name=None ++ self.socket_logger=None ++ ++ self.events={} ++ ++ try: ++ socket_dir_name = tempfile.mkdtemp(dir=self.socket_dir, ++ prefix="log_socket_") ++ ++ socket_name = os.path.join(socket_dir_name, 'socket') ++ ++ except OSError as e: ++ return ++ ++ self.socket_name=socket_name ++ ++ #print(self.socket_name) ++ ++ try: ++ socket_logger=socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) ++ socket_logger.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) ++ ++ socket_logger.bind(self.socket_name) ++ socket_logger.listen(64) ++ ++ except socket.error as e: ++ return ++ ++ self.socket_logger=socket_logger ++ ++ try: ++ # Allow connecting to socket for anyone ++ os.chmod(socket_dir_name, ++ stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR| ++ stat.S_IROTH|stat.S_IWOTH|stat.S_IXOTH) ++ os.chmod(socket_name, ++ stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR| ++ stat.S_IROTH|stat.S_IWOTH|stat.S_IXOTH) ++ except OSError as e: ++ return ++ ++ def run(self): ++ """ Starts the log server """ ++ ++ self.alive=True ++ self.listen_thread=threading.currentThread() ++ clients={} ++ ++ epoll=select.epoll() ++ epoll.register(self.socket_logger.fileno(), select.EPOLLIN) ++ ++ while self.alive: ++ try: ++ sock_events = epoll.poll(3) ++ ++ for fileno, sock_event in sock_events: ++ if fileno == self.socket_logger.fileno(): ++ ret = self.socket_logger.accept() ++ if ret is None: ++ pass ++ else: ++ (client,addr)=ret ++ epoll.register(client.fileno(), select.EPOLLIN) ++ clients[client.fileno()]=client ++ elif sock_event & select.EPOLLIN: ++ s=clients[fileno] ++ record=s.recv(8192) ++ ++ if not record: # if connection was closed ++ epoll.unregister(fileno) ++ clients[fileno].close() ++ del clients[fileno] ++ continue ++ ++ #import pdb; pdb.set_trace() ++ try: ++ message=record.decode("utf8").split("\0") ++ except UnicodeDecodeError: ++ print("Bad message %s" % record) ++ continue ++ ++ # continue ++ ++ #print(message) ++ ++ try: ++ if message[4]=="ASKING": ++ if self.filter_proc(message[1],message[2],message[3]): ++ s.sendall(b"ALLOW\0") ++ else: ++ # TODO: log through portage infrastructure ++ #print("Blocking an access to %s" % message[2]) ++ s.sendall(b"DENY\0") ++ else: ++ eventname,filename,stage,result=message[1:5] ++ ++ if not stage in self.events: ++ self.events[stage]=[{},{}] ++ ++ hashofsucesses=self.events[stage][0] ++ hashoffailures=self.events[stage][1] ++ ++ if result=="DENIED": ++ print("Blocking an access to %s" % filename) ++ ++ if result=="OK": ++ if not filename in hashofsucesses: ++ hashofsucesses[filename]=[False,False] ++ ++ readed_or_writed=hashofsucesses[filename] ++ ++ if eventname=="read": ++ readed_or_writed[0]=True ++ elif eventname=="write": ++ readed_or_writed[1]=True ++ ++ elif result[0:3]=="ERR" or result=="DENIED": ++ if not filename in hashoffailures: ++ hashoffailures[filename]=[False,False] ++ notfound_or_blocked=hashoffailures[filename] ++ ++ if result=="ERR/2": ++ notfound_or_blocked[0]=True ++ elif result=="DENIED": ++ notfound_or_blocked[1]=True ++ ++ else: ++ print("Error in logger module<->analyser protocol") ++ ++ except IndexError: ++ print("IndexError while parsing %s" % record) ++ except IOError as e: ++ if e.errno!=4: # handling "Interrupted system call" errors ++ raise ++ ++ # if main thread doesnt exists then exit ++ if not self.main_thread.is_alive(): ++ break ++ epoll.unregister(self.socket_logger.fileno()) ++ epoll.close() ++ self.socket_logger.close() ++ ++ def stop(self): ++ """ Stops the log server. Returns all events """ ++ ++ self.alive=False ++ ++ # Block the main thread until listener exists ++ self.listen_thread.join() ++ ++ # We assume portage clears tmp folder, so no deleting a socket file ++ # We assume that no new socket data will arrive after this moment ++ return self.events +diff --git a/pym/portage/const.py b/pym/portage/const.py +index ecaa8f1..f34398d 100644 +--- a/pym/portage/const.py ++++ b/pym/portage/const.py +@@ -67,6 +67,8 @@ FAKEROOT_BINARY = "/usr/bin/fakeroot" + BASH_BINARY = "/bin/bash" + MOVE_BINARY = "/bin/mv" + PRELINK_BINARY = "/usr/sbin/prelink" ++AUTODEP_LIBRARY = "/usr/lib/file_hook.so" ++ + + INVALID_ENV_FILE = "/etc/spork/is/not/valid/profile.env" + REPO_NAME_FILE = "repo_name" +@@ -88,7 +90,8 @@ EBUILD_PHASES = ("pretend", "setup", "unpack", "prepare", "configure" + SUPPORTED_FEATURES = frozenset([ + "allow-missing-manifests", + "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy", +- "ccache", "chflags", "collision-protect", "compress-build-logs", ++ "ccache", "chflags", "collision-protect", "compress-build-logs", ++ "depcheck", "depcheckstrict", + "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot", + "fail-clean", "fixpackages", "force-mirror", "getbinpkg", + "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror", +diff --git a/pym/portage/package/ebuild/_config/special_env_vars.py b/pym/portage/package/ebuild/_config/special_env_vars.py +index 87aa606..6d42809 100644 +--- a/pym/portage/package/ebuild/_config/special_env_vars.py ++++ b/pym/portage/package/ebuild/_config/special_env_vars.py +@@ -101,8 +101,8 @@ environ_whitelist += [ + # other variables inherited from the calling environment + environ_whitelist += [ + "CVS_RSH", "ECHANGELOG_USER", +- "GPG_AGENT_INFO", +- "SSH_AGENT_PID", "SSH_AUTH_SOCK", ++ "GPG_AGENT_INFO", "LOG_SOCKET", ++ "SSH_AGENT_PID", "SSH_AUTH_SOCK" + "STY", "WINDOW", "XAUTHORITY", + ] + +diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py +index 49b67ac..c76c1ed 100644 +--- a/pym/portage/package/ebuild/doebuild.py ++++ b/pym/portage/package/ebuild/doebuild.py +@@ -1038,6 +1038,9 @@ def _spawn_actionmap(settings): + nosandbox = ("sandbox" not in features and \ + "usersandbox" not in features) + ++ if "depcheck" in features or "depcheckstrict" in features: ++ nosandbox = True ++ + if not portage.process.sandbox_capable: + nosandbox = True + +@@ -1215,7 +1218,10 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero + keywords["opt_name"] = "[%s/%s]" % \ + (mysettings.get("CATEGORY",""), mysettings.get("PF","")) + +- if free or "SANDBOX_ACTIVE" in os.environ: ++ if "depcheck" in features or "depcheckstrict" in features: ++ keywords["opt_name"] += " bash" ++ spawn_func = portage.process.spawn_autodep ++ elif free or "SANDBOX_ACTIVE" in os.environ: + keywords["opt_name"] += " bash" + spawn_func = portage.process.spawn_bash + elif fakeroot: +diff --git a/pym/portage/process.py b/pym/portage/process.py +index 3c15370..6866a2f 100644 +--- a/pym/portage/process.py ++++ b/pym/portage/process.py +@@ -16,7 +16,7 @@ portage.proxy.lazyimport.lazyimport(globals(), + 'portage.util:dump_traceback', + ) + +-from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY ++from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY, AUTODEP_LIBRARY + from portage.exception import CommandNotFound + + try: +@@ -39,6 +39,9 @@ else: + sandbox_capable = (os.path.isfile(SANDBOX_BINARY) and + os.access(SANDBOX_BINARY, os.X_OK)) + ++autodep_capable = (os.path.isfile(AUTODEP_LIBRARY) and ++ os.access(AUTODEP_LIBRARY, os.X_OK)) ++ + fakeroot_capable = (os.path.isfile(FAKEROOT_BINARY) and + os.access(FAKEROOT_BINARY, os.X_OK)) + +@@ -66,6 +69,16 @@ def spawn_bash(mycommand, debug=False, opt_name=None, **keywords): + args.append(mycommand) + return spawn(args, opt_name=opt_name, **keywords) + ++def spawn_autodep(mycommand, opt_name=None, **keywords): ++ if not autodep_capable: ++ return spawn_bash(mycommand, opt_name=opt_name, **keywords) ++ if "env" not in keywords or "LOG_SOCKET" not in keywords["env"]: ++ return spawn_bash(mycommand, opt_name=opt_name, **keywords) ++ ++ # Core part: tell the loader to preload logging library ++ keywords["env"]["LD_PRELOAD"]=AUTODEP_LIBRARY ++ return spawn_bash(mycommand, opt_name=opt_name, **keywords) ++ + def spawn_sandbox(mycommand, opt_name=None, **keywords): + if not sandbox_capable: + return spawn_bash(mycommand, opt_name=opt_name, **keywords) -- cgit v1.2.3-65-gdbad