Commit dc3195b4 authored by Marcel Huber's avatar Marcel Huber
Browse files

Merge branch 'feature_9-Verify---package-behavior-when-target-is-already-built' into 'develop'

Feature 9 verify   package behavior when target is already built

See merge request !16
parents 8df1a0fe b93a5ca3
Pipeline #49058 passed with stages
in 3 minutes and 56 seconds
......@@ -10,3 +10,7 @@ dist/
.tox/
/coverage/
/.eggs/
/.wheels/
*.orig
.sconf_temp/
.sconsign*
......@@ -19,7 +19,7 @@ import functools
import itertools
import operator
from SConsider.SomeUtils import getFlatENV
from SConsider.PopenHelper import PopenHelper, ProcessRunner
from SConsider.PopenHelper import ProcessRunner
def uniquelist(iterable):
......@@ -79,9 +79,10 @@ class UnixFinder(LibFinder):
cwd=os.path.dirname(source[0].get_abspath()),
env=getFlatENV(env)) as executor:
for out, _ in executor:
for j in re.findall(r'^.*=>\s*(not found|[^\s^\(]+)', out, re.MULTILINE):
if functools.partial(operator.ne, 'not found')(j):
libs.append(j)
for (ln_lib, libpath) in re.findall(r'^\s*(.*)\s+=>\s*(not found|[^\s^\(]+)', out,
re.MULTILINE):
if functools.partial(operator.ne, 'not found')(libpath) and not ln_lib.startswith(os.sep):
libs.append(libpath)
if libnames:
libs = [j for j in libs if functools.partial(self.__filterLibs, env, libnames=libnames)(j)]
return libs
......@@ -131,9 +132,10 @@ class MacFinder(LibFinder):
cwd=os.path.dirname(source[0].get_abspath()),
env=getFlatENV(env)) as executor:
for out, _ in executor:
for j in re.findall(r'^.*=>\s*(not found|[^\s^\(]+)', out, re.MULTILINE):
if functools.partial(operator.ne, 'not found')(j):
libs.append(j)
for (ln_lib, libpath) in re.findall(r'^\s*(.*)\s+=>\s*(not found|[^\s^\(]+)', out,
re.MULTILINE):
if functools.partial(operator.ne, 'not found')(libpath) and not ln_lib.startswith(os.sep):
libs.append(libpath)
if libnames:
libs = [j for j in libs if functools.partial(self.__filterLibs, env, libnames=libnames)(j)]
......@@ -192,3 +194,51 @@ class Win32Finder(LibFinder):
def getSystemLibDirs(self, env):
return os.environ['PATH'].split(os.pathsep)
try:
from SCons.Tool import EmitLibSymlinks
except:
from SCons.Util import is_List
# copied over from scons 2.4.1
def EmitLibSymlinks(env, symlinks, libnode, **kw):
"""Used by emitters to handle (shared/versioned) library symlinks."""
nodes = list(set([x for x, _ in symlinks] + [libnode]))
clean_targets = kw.get('clean_targets', [])
if not is_List(clean_targets):
clean_targets = [clean_targets]
for link, linktgt in symlinks:
env.SideEffect(link, linktgt)
clean_list = filter(lambda x: x != linktgt, nodes)
env.Clean(list(set([linktgt] + clean_targets)), clean_list)
# consolidated copy of SCons.Tool.VersionShLibLinkNames
# and SCons.Tool.install.versionedLibVersion as of scons 2.3.6
# adapted to accept non-patch versions too
def versionedLibVersion(dest, source, env):
if (hasattr(source[0], 'attributes') and hasattr(source[0].attributes, 'shlibname')):
libname = source[0].attributes.shlibname
else:
libname = os.path.basename(str(dest))
shlib_suffix = env.subst('$SHLIBSUFFIX')
version_pattern = r'(?P<version>(?P<major>[0-9]+)(\.(?P<minor>[0-9]+)(\.(?P<patch>[0-9a-zA-Z]+))?)?)'
version = None
versioned_re = re.compile(r'(?P<libname>.*)(?P<suffix>' + re.escape(shlib_suffix) + r')\.' +
version_pattern)
result = versioned_re.search(libname)
linknames = []
if result:
version = result.group('version')
if version is not None:
# For libfoo.so.x.y.z, linknames libfoo.so libfoo.so.x.y libfoo.so.x
# First linkname has no version number
linkname = result.group('libname') + result.group('suffix')
for topdownversion in [result.group('major'), result.group('minor')]:
if topdownversion is not None:
linknames.append(linkname)
linkname = linkname + "." + topdownversion
return (version, linknames)
......@@ -73,7 +73,7 @@ def setup_main_logging(default_level=DEFAULT_LEVEL):
def ensure_prerequisites():
EnsureSConsVersion(2, 3, 0)
EnsureSConsVersion(2, 5, 0)
EnsurePythonVersion(2, 6)
......
......@@ -191,7 +191,10 @@ class PackageRegistry(object):
return packagename + targetname if packagename != targetname else targetname
@staticmethod
def collectPackageFiles(directory, match_func, file_ext='sconsider', excludes_rel=None,
def collectPackageFiles(directory,
match_func,
file_ext='sconsider',
excludes_rel=None,
excludes_abs=None):
"""Recursively collects SConsider packages.
......@@ -305,14 +308,15 @@ Original exception message:
def getPackageTargetDependencies(self, packagename, targetname, callerdeps=None):
def get_dependent_targets(pname, tname):
if hasattr(self, 'getBuildSettings'):
targetBuildSettings = self.getBuildSettings(packagename).get(targetname, {})
targetBuildSettings = self.getBuildSettings(packagename, targetname)
targetlist = targetBuildSettings.get('requires', [])
targetlist.extend(targetBuildSettings.get('linkDependencies', []))
targetlist.extend([targetBuildSettings.get('usedTarget', None)])
return [j for j in targetlist if j is not None]
else:
target = self.getPackageTarget(pname, tname)
return target.depends + target.prerequisites
prereq = target.prerequisites if target.prerequisites else []
return target.depends + prereq
def get_fulltargetname(target=None):
if isinstance(target, str):
......
......@@ -38,7 +38,7 @@ def get_config():
# _version.py
cfg = VersioneerConfig()
cfg.VCS = "git"
cfg.style = "pep440-pre"
cfg.style = "pep440-old"
cfg.tag_prefix = ""
cfg.parentdir_prefix = "SConsider-"
cfg.versionfile_source = "SConsider/_version.py"
......@@ -243,10 +243,11 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
# if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
# if there isn't one, this yields HEX[-dirty] (no NUM)
describe_out, rc = run_command(
GITS, ["describe", "--tags", "--dirty", "--always", "--long", "--match",
"%s*" % tag_prefix],
cwd=root)
describe_out, rc = run_command(GITS, [
"describe", "--tags", "--dirty", "--always", "--long", "--first-parent", "--match",
"%s*" % tag_prefix
],
cwd=root)
# --long was added in git-1.5.5
if describe_out is None:
raise NotThisMethod("'git describe' failed")
......
......@@ -22,10 +22,18 @@ from SConsider.PopenHelper import ProcessRunner
logger = getLogger(__name__)
def __get_buildsettings(registry, packagename):
if hasattr(registry, 'getBuildSettings'):
buildSettings = registry.getBuildSettings(packagename)
else:
buildSettings = {}
return buildSettings
def __get_dependencies(registry, packagename, fnobj, recursive=False):
dependencies = set()
buildSettings = registry.getBuildSettings(packagename)
buildSettings = __get_buildsettings(registry, packagename)
for _, settings in buildSettings.items():
deps = settings.get('requires', []) + settings.get('linkDependencies', [])
usedTarget = settings.get('usedTarget', '')
......@@ -68,7 +76,6 @@ def getPackageInputDirs(registry, packagename, relativeTo=None):
if not relativeTo:
relativeTo = registry.getPackageDir(packagename).get_abspath()
buildSettings = registry.getBuildSettings(packagename)
sourceDirs = set()
includeBasedir = registry.getPackageDir(packagename)
......@@ -78,11 +85,12 @@ def getPackageInputDirs(registry, packagename, relativeTo=None):
else:
return abspath
import SCons
buildSettings = __get_buildsettings(registry, packagename)
from SCons.Node import FS
for _, settings in buildSettings.items():
# directories of own cpp files
for sourcefile in settings.get('sourceFiles', []):
if not isinstance(sourcefile, SCons.Node.FS.File):
if not isinstance(sourcefile, FS.File):
sourcefile = includeBasedir.File(sourcefile)
sourceDirs.add(resolvePath(sourcefile.srcnode().dir.get_abspath(), relativeTo))
......@@ -96,7 +104,7 @@ def getPackageInputDirs(registry, packagename, relativeTo=None):
# directories of own public headers which are going to be copied
for sourcefile in settings.get('public', {}).get('includes', []):
if not isinstance(sourcefile, SCons.Node.FS.File):
if not isinstance(sourcefile, FS.File):
sourcefile = includeBasedir.File(sourcefile)
sourceDirs.add(resolvePath(sourcefile.srcnode().dir.get_abspath(), relativeTo))
......@@ -106,7 +114,7 @@ def getPackageInputDirs(registry, packagename, relativeTo=None):
def getHeaderFiles(registry, packagename):
"""Gets the header files using this package's build settings."""
headers = []
buildSettings = registry.getBuildSettings(packagename)
buildSettings = __get_buildsettings(registry, packagename)
for _, settings in buildSettings.items():
for headerFile in settings.get("public", {}).get("includes", []):
import SCons
......@@ -119,7 +127,7 @@ def getHeaderFiles(registry, packagename):
def getSourceFiles(registry, packagename):
"""Gets the source files using this package's build settings."""
sources = []
buildSettings = registry.getBuildSettings(packagename)
buildSettings = __get_buildsettings(registry, packagename)
for _, settings in buildSettings.items():
for sourcefile in settings.get('sourceFiles', []):
import SCons
......@@ -610,7 +618,8 @@ def createDoxygenAllTarget(registry):
def isExcludedPackage(packagename):
if packagename in thirdparty:
return True
for _, settings in registry.getBuildSettings(packagename).iteritems():
buildSettings = __get_buildsettings(registry, packagename)
for _, settings in buildSettings.iteritems():
if settings.get('runConfig', {}).get('type', '') == 'test':
return True
return False
......
......@@ -69,4 +69,4 @@ def generate(env):
def exists(env):
return 1
return True
......@@ -18,7 +18,7 @@ import re
import os
import threading
from logging import getLogger
from SConsider.SomeUtils import hasPathPart, isDerivedNode, multiple_replace, isFileNode, allFuncs, getNodeDependencies
from SConsider.SomeUtils import hasPathPart, isDerivedNode, multiple_replace, isFileNode, allFuncs
logger = getLogger(__name__)
# needs locking because it is manipulated during multi-threaded build phase
......@@ -48,6 +48,25 @@ def addPackageTarget(registry, buildTargets, env, destdir, **kw):
buildTargets.append(packageAliasName)
def reduceToPackageFiles(install_nodes, filters=None):
"""Reduce the list of targets to only those required for the package.
Specify additional target filters using 'filters'.
"""
if filters is None:
filters = []
if not isinstance(filters, list):
filters = [filters]
filters = [isFileNode] + filters
deps = set()
for t in install_nodes:
if allFuncs(filters, t):
deps.add(t)
return deps
def makePackage(registry, buildTargets, env, destdir, **kw):
def isNotInBuilddir(node):
return not hasPathPart(node, pathpart=env.getRelativeBuildDirectory())
......@@ -68,60 +87,61 @@ def makePackage(registry, buildTargets, env, destdir, **kw):
copyfilters = [filterBaseOutDir, filterTestsAppsGlobalsPath, filterVariantPath]
for tn in buildTargets:
if registry.isValidFulltargetname(tn):
tdeps = getTargetDependencies(
env.Alias(tn)[0], [isDerivedNode, isNotInBuilddir, isNotIncludeFile])
copyPackage(tn, tdeps, env, destdir, copyfilters)
installed_files = env.FindInstalledFiles()
reduced_targets = reduceToPackageFiles(installed_files,
[isDerivedNode, isNotInBuilddir, isNotIncludeFile])
copyPackage(tn, reduced_targets, env, destdir, copyfilters)
def copyPackage(name, deps, env, destdir, filters=None):
def copyPackage(name, deps, env, package_destdir, filters=None):
for target in deps:
copyTarget(env, determineDirInPackage(name, env, destdir, target, filters), target)
packagetarget_destdir = determineDirInPackage(name, env, package_destdir, target, filters)
copyTarget(env, packagetarget_destdir, target, package_destdir)
def install_or_link_node(env, destdir, node):
def install_node_to_destdir(targets_list, node, destdir):
def copyTarget(env, packagetarget_destdir, node, package_destdir):
old = env.Alias(packagetarget_destdir.File(node.name))
if old and old[0].sources:
if isInstalledNode(node, old[0].sources[0]) or isInstalledNode(old[0].sources[0], node):
return None
target = install_or_link_node(env, packagetarget_destdir, node, package_destdir)
env.Alias(packageAliasName, target)
return target
def install_or_link_node(env, packagetarget_destdir, node, package_destdir):
def rel_installed_path(packagetarget_destdir, package_destdir, node):
return package_destdir.rel_path(packagetarget_destdir) + os.sep + node.name
def install_node_to_destdir(targets_list, node, packagetarget_destdir, package_destdir):
from stat import S_IRUSR, S_IRGRP, S_IROTH, S_IXUSR
from SCons.Defaults import Chmod
# ensure executable flag on installed shared libs
mode = S_IRUSR | S_IRGRP | S_IROTH | S_IXUSR
node_name = node.name
if node_name in targets_list:
return targets_list[node_name]
install_path = env.makeInstallablePathFromDir(destdir)
rel_node_path = rel_installed_path(packagetarget_destdir, package_destdir, node)
if rel_node_path in targets_list:
return targets_list[rel_node_path]
install_path = env.makeInstallablePathFromDir(packagetarget_destdir)
target = env.Install(dir=install_path, source=node)
env.AddPostAction(target, Chmod(str(target[0]), mode))
targets_list[node_name] = target
targets_list[rel_node_path] = target
return target
global packageTargets, packageTargetsRLock
# build phase could be multi-threaded
with packageTargetsRLock:
# take care of already created targets otherwise we would have
# multiple ways to build the same target
global packageTargets
node_name = node.name
if node_name in packageTargets:
target = packageTargets[node_name]
else:
install_node = node
is_link = node.islink()
if is_link:
install_node = node.sources[0]
target = install_node_to_destdir(packageTargets, install_node, destdir)
if is_link:
target = env.Symlink(target[0].get_dir().File(node_name), target)
packageTargets[node_name] = target
return target
target = install_node_to_destdir(packageTargets, node, packagetarget_destdir, package_destdir)
for side_effect in node.side_effects:
rel_node_path = rel_installed_path(packagetarget_destdir, package_destdir, side_effect)
if rel_node_path not in packageTargets:
node_name = side_effect.name
ln_target = env.Symlink(packagetarget_destdir.File(node_name), target[0])
packageTargets[rel_node_path] = ln_target
target.append(ln_target)
def copyTarget(env, destdir, node):
old = env.Alias(destdir.File(node.name))
if old and old[0].sources:
if isInstalledNode(node, old[0].sources[0]) or isInstalledNode(old[0].sources[0], node):
return None
target = install_or_link_node(env, destdir, node)
env.Alias(packageAliasName, target)
return target
......@@ -208,24 +228,3 @@ def generate(env):
def exists(env):
return 1
def getTargetDependencies(target, filters=None):
"""Determines the recursive dependencies of a target (including itself).
Specify additional target filters using 'filters'.
"""
if filters is None:
filters = []
if not isinstance(filters, list):
filters = [filters]
filters = [isFileNode] + filters
deps = set()
if allFuncs(filters, target):
executor = target.get_executor()
if executor is not None:
deps.update(executor.get_all_targets())
deps.update(getNodeDependencies(target, filters))
return deps
......@@ -15,9 +15,9 @@ syntax
# -------------------------------------------------------------------------
import re
import SCons.Node
import SCons.Action
from SCons.Script import Depends, Builder
from SCons.Action import Action
from SCons.Builder import Builder
from SCons.Script import Depends
def substInFile(target, source, searchre, subfn):
......@@ -67,17 +67,20 @@ def getData(keys, env):
return data
def emit(target, source, env):
def substituted_filename_emitter(target, source, env):
from SCons.Node import FS, Python
newTarget = []
for (t, s) in zip(target, source):
if isinstance(t, SCons.Node.FS.Dir):
if isinstance(t, FS.Dir):
newTarget.append(t.File(s.name))
else:
newTarget.append(t)
keys = getKeysFromFile(str(s), getSearchRE(env))
data = getData(keys, env)
Depends(t, SCons.Node.Python.Value(data))
return newTarget, source
Depends(t, Python.Value(data))
env.AlwaysBuild(newTarget)
return (newTarget, source)
def getMarker(env):
......@@ -121,8 +124,11 @@ def substInFiles(target, source, env):
def generate(env):
substInFileAction = SCons.Action.Action(substInFiles, getLogMessage)
substInFileBuilder = Builder(action=substInFileAction, emitter=emit)
from SCons.Tool import install
substInFileAction = Action(substInFiles, getLogMessage)
substInFileBuilder = Builder(
action=substInFileAction,
emitter=[substituted_filename_emitter, install.add_targets_to_INSTALLED_FILES])
env.Append(BUILDERS={'SubstInFileBuilder': substInFileBuilder})
......
......@@ -14,11 +14,13 @@ Tool to collect system libraries needed by an executable/shared library
# -------------------------------------------------------------------------
import os
import re
import threading
from logging import getLogger
from SCons.Errors import UserError
from SCons.Node.Alias import default_ans
from SConsider.LibFinder import FinderFactory
from SConsider.LibFinder import FinderFactory, EmitLibSymlinks, versionedLibVersion
from SCons.Tool import install as inst_tool
logger = getLogger(__name__)
# needs locking because it is manipulated during multi-threaded build phase
......@@ -45,17 +47,27 @@ def get_libdirs(env, ownlibDir, finder):
return libdirs
def get_dependent_libs(env, sourcenode, libdirs_func=get_libdirs):
ownlibDir = get_library_install_dir(env, sourcenode)
def get_dependent_libs(env, sourcenode, library_install_dir, libdirs_func=get_libdirs):
finder = FinderFactory.getForPlatform(env["PLATFORM"])
libdirs = libdirs_func(env, ownlibDir, finder)
libdirs = libdirs_func(env, library_install_dir, finder)
return finder.getLibs(env, [sourcenode], libdirs=libdirs)
def real_lib_path(env, target):
node = target
while node.islink():
if node.sources:
node = node.sources[0]
else:
node = env.File(os.path.realpath(node.get_abspath()))
return node
def installSystemLibs(source):
"""This function is called during the build phase and adds targets
dynamically to the dependency tree."""
from SConsider.PackageRegistry import PackageRegistry
from SCons.Defaults import SharedObjectEmitter
sourcenode = PackageRegistry().getRealTarget(source)
if not sourcenode:
return None
......@@ -63,54 +75,52 @@ def installSystemLibs(source):
env = sourcenode.get_env()
ownlibDir = get_library_install_dir(env, sourcenode)
deplibs = get_dependent_libs(env, sourcenode)
deplibs = get_dependent_libs(env, sourcenode, ownlibDir)
# don't create cycles by copying our own libs
# but don't mask system libs
deplibs = [env.File(j) for j in deplibs if notInDir(env, ownlibDir, j)]
source_syslibs = []
global systemLibTargets, systemLibTargetsRLock
def install_node_to_destdir(targets_list, node, destdir):
from stat import S_IRUSR, S_IRGRP, S_IROTH, S_IXUSR
from SCons.Defaults import Chmod
# ensure executable flag on installed shared libs
mode = S_IRUSR | S_IRGRP | S_IROTH | S_IXUSR
node_name = node.name
if node_name in targets_list:
return targets_list[node_name]
install_path = env.makeInstallablePathFromDir(destdir)
# make sure we do not install over an own node
if env.Dir(install_path).File(node.name).has_builder():
return None
target = env.Install(dir=install_path, source=node)
env.AddPostAction(target, Chmod(str(target[0]), mode))
targets_list[node_name] = target
def install_node_to_destdir(targets_list, node, install_path, fn=env.Install):
target = fn(dir=install_path, source=node)
targets_list[node.name] = target
return target
global systemLibTargets, systemLibTargetsRLock
# build phase could be multi-threaded
with systemLibTargetsRLock:
for node in deplibs:
install_dir = env.makeInstallablePathFromDir(ownlibDir)
for libnode in deplibs:
real_libnode = real_lib_path(env, libnode)
# tag file node as shared library
real_libnode, _ = SharedObjectEmitter([real_libnode], None, None)
real_libnode = real_libnode[0]
node_name = real_libnode.name
target = []
node_name = node.name
if node_name in systemLibTargets:
target = systemLibTargets[node_name]
else:
install_node = node
is_link = node.islink()
if is_link:
if node.sources:
install_node = node.sources[0]
else:
install_node = env.File(os.path.realpath(node.get_abspath()))
if not install_node.is_under(ownlibDir):
target = install_node_to_destdir(systemLibTargets, install_node, ownlibDir)
# do not create another node with the same name in this case
# /usr/lib/gcc/x86_64-linux-gnu/9/32/libgcc_s.so.1 -> ../../../../../lib32/libgcc_s.so.1
if target and is_link and not node_name == install_node.name:
target = env.Symlink(target[0].get_dir().File(node_name), target)
systemLibTargets[node_name] = target
# figure out if we deal with a versioned shared library
# otherwise we need to fall back to Install builder and Symlink
version, linknames = versionedLibVersion(real_libnode, source, env)
if version:
symlinks = map(lambda n: (env.fs.File(n, install_dir), real_libnode), linknames)
EmitLibSymlinks(env, symlinks, real_libnode)
real_libnode.attributes.shliblinks = symlinks
target = install_node_to_destdir(systemLibTargets,
real_libnode,
install_dir,
fn=env.InstallVersionedLib)
else:
target = install_node_to_destdir(systemLibTargets, real_libnode, install_dir)
if not node_name == libnode.name:
fulllinkname = os.path.join(install_dir, libnode.name)
ln_target = env.Symlink(fulllinkname, target[0])
target.extend(ln_target)
for linkname in linknames:
systemLibTargets[linkname] = target
if target and not target[0] in source_syslibs:
source_syslibs.extend(target)
......@@ -141,29 +151,34 @@ def generate(env, *args, **kw):
doesn't exist..."""