Skip to content

GitLab

Commit ae82cb0f authored by Marcel Huber's avatar Marcel Huber
Browse files

improved openssl certificate setup

parent 773d1c4b
Pipeline #48743 passed with stages
in 21 minutes and 37 seconds
Hide whitespace changes
Inline Side-by-side
coast/modules/SSL/Test/CoastSSLTest.sconsider
View file @ ae82cb0f
......@@ -21,15 +21,16 @@ from OpenSSL import SSL, crypto
import Anything
import ServerExtensions
from SomeUtils import runCommand, replaceRegexInFile, listFiles, findFiles, removeFiles, getfqdn
from RunBuilder import SkipTest
Import('*')
defaults = None
defaults = Anything.Anything()
_sconsider_dist = pkg_resources.get_distribution("SConsider").parsed_version
_helperscript_target = 'openssl.script_openssl'
_openssl_script_target = 'openssl.script_openssl'
if _sconsider_dist < pkg_resources.parse_version("0.3.11"):
_helperscript_target = 'openssl.openSSL'
_openssl_script_target = 'openssl.openSSL'
def getRealTarget(fulltargetname):
from SConsider.PackageRegistry import splitFulltargetname
......@@ -39,6 +40,24 @@ if _sconsider_dist < pkg_resources.parse_version("0.3.11"):
if not hasattr(registry, 'getRealTarget'):
registry.getRealTarget = getRealTarget
if _sconsider_dist < pkg_resources.parse_version("0.3.10"):
from SConsider.PackageRegistry import createFulltargetname
else:
from SConsider.PackageRegistry import PackageRegistry
createFulltargetname = PackageRegistry.createFulltargetname
logpathrel = 'logs'
rotatepathrel = 'rotate'
configdir = 'config'
certdir = 'cert_files'
certfilename = packagename
certsuffix = '.pem'
configdirs = [
configdir,
certdir,
'.',
]
servers = []
hostname, domain, fqdn = getfqdn()
searchReplaceDict = {
......@@ -46,6 +65,10 @@ searchReplaceDict = {
'HOSTNAME': hostname,
'HOSTNAMEIP': socket.gethostbyname(hostname),
'DOMAINONLY': domain,
'LOGDIRREL': logpathrel,
'ROTATEDIRREL': lambda env: setAndCreateRotatedir(env),
'COAST_PATH': ':'.join(configdirs),
'COAST_ROOT': lambda env: env.getTargetBaseInstallDir().abspath,
}
......@@ -68,42 +91,18 @@ def appendReplaceCommandForCert(key, filename, searchReplace):
pass
def ModifyConfigFiles(env, searchReplace=[], files=[]):
for file in files:
fname = file.abspath
if os.path.isfile(fname):
oldmode = os.stat(fname).st_mode
# set writable
os.chmod(fname, oldmode | S_IWUSR)
# replace tokens in file
replaceRegexInFile(fname, searchReplace)
os.chmod(fname, oldmode)
servers = []
def startServer(host, port, server_class, handler_class, **kw):
server = server_class((host, port), handler_class, **kw)
servers.append(server)
server.serve_forever()
def prepareMkCaParams(env, commandargs, tdirabs, certdir):
mkcaScriptName = os.path.join(tdirabs, 'mkca-new.sh')
def prepareMkCaParams(env, commandargs, tdirabs, certOutDir, mkcaScriptName, cnfFilename=''):
# set writable cfgfilename
target = registry.getRealTarget(_helperscript_target)
target = registry.getRealTarget(_openssl_script_target)
if not os.path.isfile(mkcaScriptName):
return False
commandargs.extend([mkcaScriptName])
if target:
commandargs.extend(['-b', target.abspath])
commandargs.extend(['-B', '-S --'])
cnfFilename = os.path.join(tdirabs, 'openssl.cnf')
if cnfFilename:
commandargs.extend(['-o', cnfFilename])
mkcertDirectory = os.path.join(tdirabs, 'mkcertdir')
certOutDir = os.path.join(tdirabs, certdir)
for theDir in [mkcertDirectory, certOutDir]:
if os.path.isdir(theDir):
shutil.rmtree(theDir, ignore_errors=True)
......@@ -112,38 +111,45 @@ def prepareMkCaParams(env, commandargs, tdirabs, certdir):
# maybe we should do the following step by hand because we do not need
# cert suffixes
commandargs.extend(['-c', certOutDir])
commandargs.extend(['-C', defaults['HostName']])
commandargs.extend(['-S', defaults['HostName']])
fqdn = defaults.get('SSLHost', '')
if fqdn and fqdn != defaults['HostName']:
commandargs.extend(['-S', fqdn])
def createCertificates(env):
tdir = env.getTargetBaseInstallDir()
_hostname = defaults.get('HostName', hostname)
_fqdn = defaults.get('SSLHost', fqdn)
commandargs.extend(['-C', _hostname])
commandargs.extend(['-S', _hostname])
if _fqdn and _fqdn != _hostname:
commandargs.extend(['-S', _fqdn])
def createCertificates(env,
targetdir,
mkcaScriptName,
certdirrel='cert_files',
createlogdirrel='.',
cnfFilename='',
doCertDict=True):
tdirabs = targetdir.abspath
certOutDir = os.path.join(tdirabs, certdirrel)
commandargs = []
stdincontent = []
certdir = 'cert_files'
certOutDir = os.path.join(tdir.abspath, certdir)
prepareMkCaParams(env, commandargs, tdir.abspath, certdir)
cnfFilename = os.path.join(tdir.abspath, 'openssl.cnf')
logpath = env.getLogInstallDir()
retCode = runCommand(commandargs,
logpath=logpath.abspath,
stdincontent='\n'.join(stdincontent),
timeout=20.0)
prepareMkCaParams(env, commandargs, tdirabs, certOutDir, mkcaScriptName, cnfFilename)
logpathabs = targetdir.Dir(createlogdirrel).abspath
certFileNodesList = []
certDict = {}
if retCode == 0:
retCode = 1
if commandargs:
retCode = runCommand(commandargs,
logpath=logpathabs,
stdincontent='\n'.join(stdincontent),
timeout=20.0)
if retCode == 0 and doCertDict:
# get list of copied certificates
certFileNodesList = findFiles([certOutDir], ['.pem', '.p12'])
certDict = createCertDict(certFileNodesList)
target = registry.getRealTarget(_helperscript_target)
target = registry.getRealTarget(_openssl_script_target)
sslbin = 'openssl'
if target:
sslbin = '' + target.abspath + ' -S --'
# if there is now c_rehash available, use the following improved
# script, (base from from
# script, (based on
# http://www.commandlinefu.com/commands/view/2532/c_rehash-replacement)
command = r"""
for file in *crt*.pem; do
......@@ -151,7 +157,7 @@ for file in *crt*.pem; do
test -f ${theHash}.0 && true || ln -s $file ${theHash}.0;
done"""
retCode = runCommand(command,
logpath=logpath.abspath,
logpath=logpathabs,
stdincontent=None,
shell=True,
cwd=certOutDir,
......@@ -166,10 +172,10 @@ def createCertDict(certFileNodesList):
'chainwoservercrt', 'subCA1crt', 'subCA1key', 'rootCA1crt', 'rootCA1key'
]:
try:
_hostname = defaults.get('HostName', hostname)
filename = [
c.abspath for c in certFileNodesList
if re.match(r'' + cname + '(_' + defaults['HostName'] + ')?\.pem',
os.path.split(c.abspath)[1])
c.abspath for c in certFileNodesList if re.match(r'' + cname + '(_' + _hostname + ')?\.pem',
os.path.split(c.abspath)[1])
][0]
certDict[cname] = filename
except:
......@@ -177,9 +183,39 @@ def createCertDict(certFileNodesList):
return certDict
def setAndCreateRotatedir(env):
targetdir = env.getTargetBaseInstallDir()
logrotatepathrel = os.path.join(logpathrel, rotatepathrel)
logdir = targetdir.Dir(logrotatepathrel)
if not os.path.isdir(logdir.abspath):
os.makedirs(logdir.abspath)
return logrotatepathrel
def ModifyConfigFiles(env, searchReplace=[], files=[]):
for file in files:
fname = file.abspath
if os.path.isfile(fname):
oldmode = os.stat(fname).st_mode
# set writable
os.chmod(fname, oldmode | S_IWUSR)
# replace tokens in file
replaceRegexInFile(fname, searchReplace)
os.chmod(fname, oldmode)
def startServer(host, port, server_class, handler_class, **kw):
server = server_class((host, port), handler_class, **kw)
servers.append(server)
server.serve_forever()
def setUp(target, source, env):
tdir = env.getTargetBaseInstallDir()
Anything.setLocalEnv(env={}, COAST_ROOT=tdir.abspath)
targetdir = env.getTargetBaseInstallDir()
logdirrel = targetdir.rel_path(env.getLogInstallDir())
tdirabs = targetdir.abspath
configdirabs = targetdir.Dir(configdir).abspath
Anything.setLocalEnv(env={}, COAST_ROOT=tdirabs)
global defaults
try:
......@@ -191,45 +227,71 @@ def setUp(target, source, env):
'setup tests,\ncheck COAST_ROOT/COAST_PATH setting of ' \
'Anything!')
tdirabs = tdir.abspath
logpath = env.getLogInstallDir()
configdir = tdir.Dir('config')
# env['ENV']['COAST_ROOT'] = tdirabs
env['ENV']['COAST_LOGONCERR'] = '3'
# env['ENV']['COAST_PATH'] = ':'.join(['config', '.'])
# create certificate files
certOutDir, certFileNodesList, certDict = createCertificates(env)
if certDict:
env['ENV']['COAST_ROOT'] = tdirabs
env['ENV']['COAST_PATH'] = ':'.join(configdirs)
env['ENV']['COAST_TRACE_INITFINIS'] = 0
env['ENV']['COAST_TRACE_STORAGE'] = 1
env['ENV']['COAST_LOGONCERR'] = 5 # ignore ERRORS and below
env['ENV']['COAST_USE_MMAP_STREAMS'] = 0
#setAndCreateRotatedir(env)
if True or _sconsider_dist < pkg_resources.parse_version("0.5"):
# create certificate files
sslconfpath = ''
mkcaScriptName = ''
searchReplace = []
for key, value in certDict.items():
searchReplace.append((r"##" + key + "_abs##", value))
searchReplace.append((r"##" + key + "_rel##", os.path.relpath(value, tdirabs)))
if key.endswith('crt'):
appendReplaceCommandForCert(key, value, searchReplace)
searchReplace.append((r"##certdir_abs##", certOutDir))
searchReplace.append((r"##certdir_rel##", os.path.relpath(certOutDir, tdirabs)))
ModifyConfigFiles(env, searchReplace, findFiles([configdir], ['.any']))
threading.Thread(target=startServer,
args=[
defaults['SSLHost'], defaults['SSLPort'], ServerExtensions.SecureHTTPServer,
ServerExtensions.SecureHTTPRequestHandler
],
kwargs={
'certFile': certDict.get('servercrt'),
'keyFile': certDict.get('serverkey'),
'caChainFile': certDict.get('chainwoservercrt')
}).start()
threading.Thread(target=startServer,
args=[
defaults['SSLHighCipherHost'], defaults['SSLHighCipherPort'],
ServerExtensions.SecureHTTPServer, ServerExtensions.SecureHTTPRequestHandler
],
kwargs={
'certFile': certDict.get('servercrt'),
'keyFile': certDict.get('serverkey'),
'caChainFile': certDict.get('chainwoservercrt'),
'ciphers': defaults['SSLHighCipherList']
}).start()
filelist = findFiles([configdirabs], matchfiles=['openssl.cnf'])
if filelist:
sslconfpath = filelist[0].abspath
filelist = findFiles([configdirabs], matchfiles=['mkca-new.sh'])
if filelist:
mkcaScriptName = filelist[0].abspath
if mkcaScriptName:
certOutDir, certFileNodesList, certDict = createCertificates(env,
targetdir=targetdir,
mkcaScriptName=mkcaScriptName,
certdirrel=certdir,
createlogdirrel=logdirrel,
cnfFilename=sslconfpath,
doCertDict=True)
if certDict:
for key, value in certDict.items():
searchReplace.append((r"##" + key + "_abs##", value))
searchReplace.append((r"##" + key + "_rel##", os.path.relpath(value, tdirabs)))
if key.endswith('crt'):
appendReplaceCommandForCert(key, value, searchReplace)
searchReplace.append((r"##certdir_abs##", certOutDir))
searchReplace.append((r"##certdir_rel##", os.path.relpath(certOutDir, tdirabs)))
ModifyConfigFiles(env, searchReplace, findFiles([configdirabs], ['.any', '.sh']))
threading.Thread(target=startServer,
args=[
defaults['SSLHost'], defaults['SSLPort'],
ServerExtensions.SecureHTTPServer,
ServerExtensions.SecureHTTPRequestHandler
],
kwargs={
'certFile': certDict.get('servercrt'),
'keyFile': certDict.get('serverkey'),
'caChainFile': certDict.get('chainwoservercrt')
}).start()
threading.Thread(target=startServer,
args=[
defaults['SSLHighCipherHost'], defaults['SSLHighCipherPort'],
ServerExtensions.SecureHTTPServer,
ServerExtensions.SecureHTTPRequestHandler
],
kwargs={
'certFile': certDict.get('servercrt'),
'keyFile': certDict.get('serverkey'),
'caChainFile': certDict.get('chainwoservercrt'),
'ciphers': defaults['SSLHighCipherList']
}).start()
return
raise SkipTest('Failed to create certificates to run tests!')
def tearDown(target, source, env):
......@@ -241,16 +303,20 @@ def tearDown(target, source, env):
programDepends = [
'testfwWDBase.testfwWDBase',
'CoastSSL.CoastSSL',
createFulltargetname('testfwWDBase', default=True),
createFulltargetname('CoastSSL', default=True),
]
scriptRequireNodes = [_helperscript_target]
sourceFiles = listFiles(['*.cpp'])
scriptRequireNodes = [_openssl_script_target]
includeSubdir = ''
sourceFiles = listFiles([os.path.join(includeSubdir, '*.cpp')])
exportIncludes = listFiles([os.path.join(includeSubdir, '*.h')])
copyFiles = [
(findFiles(['.'], extensions=['.any']), S_IRUSR | S_IRGRP | S_IROTH, searchReplaceDict),
(findFiles(['.'], matchfiles=['openssl.cnf']), S_IRUSR | S_IRGRP | S_IROTH),
(findFiles(['.'], matchfiles=['mkca-new.sh']), S_IRUSR | S_IRGRP | S_IROTH | S_IXUSR),
(findFiles([configdir], extensions=['.any', '.cnf']), S_IRUSR | S_IRGRP | S_IROTH, searchReplaceDict),
(findFiles([configdir], extensions=['.sh']), S_IRUSR | S_IRGRP | S_IROTH | S_IXUSR, searchReplaceDict),
]
runConfig = {
'setUp': setUp,
'tearDown': tearDown,
......@@ -258,12 +324,13 @@ runConfig = {
if _sconsider_dist < pkg_resources.parse_version("0.5"):
from SConsider import createTargets
buildSettings = {
packagename: {
'targetType': 'ProgramTest',
'includeSubdir': includeSubdir,
'linkDependencies': programDepends,
'requires': scriptRequireNodes,
'includeSubdir': '',
'sourceFiles': sourceFiles,
'copyFiles': copyFiles,
'runConfig': runConfig,
......@@ -271,6 +338,7 @@ if _sconsider_dist < pkg_resources.parse_version("0.5"):
}
createTargets(packagename, buildSettings)
elif _sconsider_dist >= pkg_resources.parse_version("0.5"):
from TargetMaker import EnvBuilder
from TargetHelpers import createCopyFilesTarget, createPackageTestTarget, createPackageProgram, createPackageProgramScript, createAlias
......@@ -297,7 +365,7 @@ elif _sconsider_dist >= pkg_resources.parse_version("0.5"):
scriptTarget = createPackageProgramScript(packagename,
bin_name,
bin_target,
depend=copyTarget,
depend=[copyTarget],
require=scriptRequireNodes,
pkg_local=[setRelTargetDir])
scripts = EnvBuilder().depend(scriptTarget).build().RegisterPackageTarget(packagename=packagename,
......
coast/modules/StdDataAccess/Test/CoastStdDataAccessTest.sconsider
View file @ ae82cb0f
......@@ -23,16 +23,17 @@ from BaseHTTPServer import HTTPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler
import Anything
import ServerExtensions
from SomeUtils import runCommand, replaceRegexInFile, listFiles, findFiles, removeFiles
from SomeUtils import runCommand, replaceRegexInFile, listFiles, findFiles, removeFiles, getfqdn
from RunBuilder import SkipTest
Import('*')
defaults = None
defaults = Anything.Anything()
_sconsider_dist = pkg_resources.get_distribution("SConsider").parsed_version
_helperscript_target = 'openssl.script_openssl'
_openssl_script_target = 'openssl.script_openssl'
if _sconsider_dist < pkg_resources.parse_version("0.3.11"):
_helperscript_target = 'openssl.openSSL'
_openssl_script_target = 'openssl.openSSL'
def getRealTarget(fulltargetname):
from SConsider.PackageRegistry import splitFulltargetname
......@@ -42,36 +43,36 @@ if _sconsider_dist < pkg_resources.parse_version("0.3.11"):
if not hasattr(registry, 'getRealTarget'):
registry.getRealTarget = getRealTarget
class SMTPThread(threading.Thread):
def __init__(self, host, port, path, logfile):
super(SMTPThread, self).__init__()
self.smtpd = ServerExtensions.SMTPFileSinkServer((host, port), None, path, logfile)
self.daemon = True
def run(self):
asyncore.loop()
class MyHTTPServer(HTTPServer):
"""This class wrapper is needed to set allow_reuse_address to True prior to
initializing it."""
allow_reuse_address = True
class HTTPEchoReplier(SimpleHTTPRequestHandler):
def _set_headers(self):
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
def do_POST(self):
# Doesn't do anything with posted data
content_length = int(self.headers['Content-Length']) # <--- Gets the size of data
post_data = self.rfile.read(content_length) # <--- Gets the data itself
self._set_headers()
self.wfile.write(post_data)
print self.headers
if _sconsider_dist < pkg_resources.parse_version("0.3.10"):
from SConsider.PackageRegistry import createFulltargetname
else:
from SConsider.PackageRegistry import PackageRegistry
createFulltargetname = PackageRegistry.createFulltargetname
logpathrel = 'logs'
rotatepathrel = 'rotate'
configdir = 'config'
certdir = 'cert_files'
certfilename = packagename
certsuffix = '.pem'
configdirs = [
configdir,
certdir,
'.',
]
servers = []
hostname, domain, fqdn = getfqdn()
searchReplaceDict = {
'FQDN': fqdn,
'HOSTNAME': hostname,
'HOSTNAMEIP': socket.gethostbyname(hostname),
'DOMAINONLY': domain,
'LOGDIRREL': logpathrel,
'ROTATEDIRREL': lambda env: setAndCreateRotatedir(env),
'COAST_PATH': ':'.join(configdirs),
'COAST_ROOT': lambda env: env.getTargetBaseInstallDir().abspath,
}
def appendReplaceCommandForCert(key, filename, searchReplace):
......@@ -93,42 +94,18 @@ def appendReplaceCommandForCert(key, filename, searchReplace):
pass
def ModifyConfigFiles(env, searchReplace=[], files=[]):
for file in files:
fname = file.abspath
if os.path.isfile(fname):
oldmode = os.stat(fname).st_mode
# set writable
os.chmod(fname, oldmode | S_IWUSR)
# replace tokens in file
replaceRegexInFile(fname, searchReplace)
os.chmod(fname, oldmode)
servers = []
def startServer(host, port, server_class, handler_class, **kw):
server = server_class((host, port), handler_class, **kw)
servers.append(server)
server.serve_forever()
def prepareMkCaParams(env, commandargs, tdirabs, certdir):
mkcaScriptName = os.path.join(tdirabs, 'mkca-new.sh')
def prepareMkCaParams(env, commandargs, tdirabs, certOutDir, mkcaScriptName, cnfFilename=''):
# set writable cfgfilename
target = registry.getRealTarget(_helperscript_target)
target = registry.getRealTarget(_openssl_script_target)
if not os.path.isfile(mkcaScriptName):
return False
commandargs.extend([mkcaScriptName])
if target:
commandargs.extend(['-b', target.abspath])
commandargs.extend(['-B', '-S --'])
cnfFilename = os.path.join(tdirabs, 'openssl.cnf')
if cnfFilename:
commandargs.extend(['-o', cnfFilename])
mkcertDirectory = os.path.join(tdirabs, 'mkcertdir')
certOutDir = os.path.join(tdirabs, certdir)
for theDir in [mkcertDirectory, certOutDir]:
if os.path.isdir(theDir):
shutil.rmtree(theDir, ignore_errors=True)
......@@ -137,38 +114,45 @@ def prepareMkCaParams(env, commandargs, tdirabs, certdir):
# maybe we should do the following step by hand because we do not need
# cert suffixes
commandargs.extend(['-c', certOutDir])
commandargs.extend(['-C', defaults['HostName']])
commandargs.extend(['-S', defaults['HostName']])
fqdn = defaults.get('SSLHost', '')
if fqdn and fqdn != defaults['HostName']:
commandargs.extend(['-S', fqdn])
def createCertificates(env):
tdir = env.getTargetBaseInstallDir()
_hostname = defaults.get('HostName', hostname)
_fqdn = defaults.get('SSLHost', fqdn)
commandargs.extend(['-C', _hostname])
commandargs.extend(['-S', _hostname])
if _fqdn and _fqdn != _hostname:
commandargs.extend(['-S', _fqdn])
def createCertificates(env,
targetdir,
mkcaScriptName,
certdirrel='cert_files',
createlogdirrel='.',
cnfFilename='',
doCertDict=True):
tdirabs = targetdir.abspath
certOutDir = os.path.join(tdirabs, certdirrel)
commandargs = []
stdincontent = []
certdir = 'cert_files'
certOutDir = os.path.join(tdir.abspath, certdir)
prepareMkCaParams(env, commandargs, tdir.abspath, certdir)
cnfFilename = os.path.join(tdir.abspath, 'openssl.cnf')
logpath = env.getLogInstallDir()
retCode = runCommand(commandargs,
logpath=logpath.abspath,
stdincontent='\n'.join(stdincontent),
timeout=20.0)
prepareMkCaParams(env, commandargs, tdirabs, certOutDir, mkcaScriptName, cnfFilename)
logpathabs = targetdir.Dir(createlogdirrel).abspath
certFileNodesList = []
certDict = {}
if retCode == 0:
retCode = 1
if commandargs:
retCode = runCommand(commandargs,
logpath=logpathabs,
stdincontent='\n'.join(stdincontent),
timeout=20.0)
if retCode == 0 and doCertDict:
# get list of copied certificates
certFileNodesList = findFiles([certOutDir], ['.pem', '.p12'])
certDict = createCertDict(certFileNodesList)
target = registry.getRealTarget(_helperscript_target)
target = registry.getRealTarget(_openssl_script_target)
sslbin = 'openssl'
if target:
sslbin = '' + target.abspath + ' -S --'
# if there is now c_rehash available, use the following improved
# script, (base from from
# script, (based on
# http://www.commandlinefu.com/commands/view/2532/c_rehash-replacement)
command = r"""
for file in *crt*.pem; do
......@@ -176,7 +160,7 @@ for file in *crt*.pem; do
test -f ${theHash}.0 && true || ln -s $file ${theHash}.0;
done"""
retCode = runCommand(command,
logpath=logpath.abspath,
logpath=logpathabs,
stdincontent=None,
shell=True,
cwd=certOutDir,
......@@ -191,10 +175,10 @@ def createCertDict(certFileNodesList):
'chainwoservercrt', 'subCA1crt', 'subCA1key', 'rootCA1crt', 'rootCA1key'
]:
try:
_hostname = defaults.get('HostName', hostname)
filename = [
c.abspath for c in certFileNodesList
if re.match(r'' + cname + '(_' + defaults['HostName'] + ')?\.pem',
os.path.split(c.abspath)[1])
c.abspath for c in certFileNodesList if re.match(r'' + cname + '(_' + _hostname + ')?\.pem',
os.path.split(c.abspath)[1])
][0]
certDict[cname] = filename
except:
......@@ -202,9 +186,70 @@ def createCertDict(certFileNodesList):
return certDict
def setAndCreateRotatedir(env):
targetdir = env.getTargetBaseInstallDir()
logrotatepathrel = os.path.join(logpathrel, rotatepathrel)