Prepared exercises for the following two weeks

Merge branch 'release_20.8.1'

.gitignore

@@ -6,3 +6,4 @@ blub*
 *.log
 .*.html
 stem-*
+node_modules/

ColumnStore/.monetdb-ds2

+user=ds2
+password=ds2
+

ColumnStore/.monetdb-monetdb

+user=monetdb
+password=monetdb
+

ColumnStore/Queries_MonetDB.sh

@@ -5,41 +5,56 @@ _EX_DIR="$(realpath --relative-to=. $_MY_DIR)"
 test -z "$_EX_DIR" && _EX_DIR=.
 
 eo0=""
-eo1="trace"
-table_suffix="${1}"
+eo1=""
 
 source $_EX_DIR/Queries_common.sh
-
+# disable globbing for star queries (*)
+set -f
 cat <<EOF
+-- Source this file from within mclient using \< Queries_MonetDB.sql
+-- If "\f trash" does not work (usually when using content from stdin), set
+-- "--interactive" option when starting mclient
+-- disable auto commit
+\a
+-- initially disable performance timing (trace command)
+\t none
+set schema ds2;
+-- FIXME: set correct permissions to select from sys.tracelog as ds2 user
+set role sysadmin;
+--
 $(for q in "${queries[@]}"; do
     explainopts="$(eval echo $(echo \"\$$(cut -d'|' -f1 <<<$q)\"))";
     aufgabe="$(cut -d'|' -f2 <<<$q)";
     aufgabe_desc="$(cut -d'|' -f3 <<<$q)";
     query="$(cut -d'|' -f4 <<<$q)";
+    query_escaped="$(echo -n $query | sed 's/\x27/\\\x27/g')";
     baseindex="";
     indexcond="";
-    echo "${echostr}--== Query $aufgabe_desc ==--"
-    echo "${echostr}-- tag::${aufgabe}[]"
-    echo "\! echo \"$explainopts\" >/tmp/qa"
-    echo "${echostr}-- tag::${aufgabe}-eo[]"
-    echo "\! cat /tmp/qa"
-    echo "${echostr}-- end::${aufgabe}-eo[]"
-    echo "\! echo \"$query\" | pg_format -f1 -s2 -u2 | tee -a /tmp/qa >/tmp/q"
-    echo "${echostr}-- tag::${aufgabe}-query[]"
-    echo "\! cat /tmp/q"
-    echo "${echostr}-- end::${aufgabe}-query[]"
-    echo "BEGIN;"
-    echo "${echostr}-- tag::${aufgabe}-explain[]"
-    echo "\pset format asciidoc"
-    echo "${echostr}-- tag::${aufgabe}-explain-adoc[]"
-    echo "\i /tmp/qa"
-    echo "${echostr}-- end::${aufgabe}-explain-adoc[]"
-    echo "\pset format ${format_text}"
-    echo "${echostr}-- tag::${aufgabe}-explain-text[]"
-    echo "\i /tmp/qa"
-    echo "${echostr}-- end::${aufgabe}-explain-text[]"
-    echo "${echostr}-- end::${aufgabe}-explain[]"
-    echo "ROLLBACK;"
-    echo "${echostr}-- end::${aufgabe}[]"
+    echo "\f tab"
+    echo "select '--== Query "$aufgabe_desc" ==--';"
+    echo "select '-- tag::"${aufgabe}"[]';"
+    echo "select '-- tag::"${aufgabe}"-query[]';"
+    echo "select '"$query_escaped"';"
+    echo "select '-- end::"${aufgabe}"-query[]';"
+    echo "select '-- tag::"${aufgabe}"-plan[]';"
+    echo "\f raw"
+    echo "plan $query"
+    echo "\f tab"
+    echo "select '-- end::"${aufgabe}"-plan[]';"
+    echo "select '-- tag::"${aufgabe}"-timing[]';"
+    echo "\f trash"
+    echo "\t performance"
+    echo "$query"
+    echo "\t none"
+    echo "\f tab"
+    echo "select '-- end::"${aufgabe}"-timing[]';"
+    echo "select '-- tag::"${aufgabe}"-trace[]';"
+    echo "\f rowcount"
+    echo "trace $query"
+    echo "\f csv+;"
+    echo "select ticks as usec, stmt from sys.tracelog;"
+    echo "\f tab"
+    echo "select '-- end::"${aufgabe}"-trace[]';"
+    echo "select '-- end::"${aufgabe}"[]';"
 done)
 EOF

ColumnStore/Queries_PostgreSQL.sh

@@ -4,9 +4,11 @@ _MY_DIR="$(cd $(dirname ${0:-.}) && pwd)"
 _EX_DIR="$(realpath --relative-to=. $_MY_DIR)"
 test -z "$_EX_DIR" && _EX_DIR=.
 
+db_name=${1:-ds2}
+db_user=${2:-$db_name}
+
 eo0=""
 eo1="EXPLAIN (ANALYZE 1,COSTS 0,TIMING 1,BUFFERS 0,VERBOSE 0,FORMAT TEXT)"
-table_suffix="${1}"
 
 source $_EX_DIR/Queries_common.sh
 
@@ -18,6 +20,7 @@ colwidth=0
 cat <<EOF
 -- anzeigen der aktuellen Einstellungen
 -- siehe auch http://www.postgresql.org/docs/current/static/runtime-config-query.html#RUNTIME-CONFIG-QUERY-ENABLE
+\c postgres postgres
 SHOW enable_seqscan;
 SHOW seq_page_cost;
 SHOW random_page_cost;
@@ -28,8 +31,9 @@ SET random_page_cost TO 1.0;
 SET track_counts TO false;
 \pset tuples_only on
 \set ECHO none
--- columns is effective when using wrapped mode
+-- columns is in effect when using wrapped mode
 \pset columns $colwidth
+\c $db_name $db_user
 
 $(for q in "${queries[@]}"; do
     explainopts="$(eval echo $(echo \"\$$(cut -d'|' -f1 <<<$q)\"))";

ColumnStore/Queries_common.sh

@@ -21,15 +21,15 @@ queries=(
 "eo1|count-orderlines-lineid_gt6|(Q20)|SELECT COUNT(*) Anzahl FROM orderlines WHERE orderlineid > 6;|orderlines(orderid,orderlineid)|"
 "eo1|count-quantity_sum-orderlines-lineid_gt6|(Q21)|SELECT COUNT(*) Anzahl, SUM(quantity) Menge FROM orderlines WHERE orderlineid > 6;|orderlines(orderid,orderlineid)|orderlines(orderlineid,quantity)"
 "eo1|count-quantity_sum-orderlines-lineid_gt10|(Q22)|SELECT COUNT(*) Anzahl, SUM(quantity) Menge FROM orderlines WHERE orderlineid > 10;|orderlines(orderid,orderlineid)|orderlines(orderlineid,quantity)"
-"eo1|star-products-titlelike|(Q24)|select * from products where title like '%ACADEMY FLY%';||products(title)@products using gist(title gist_trgm_ops)"
-"eo1|simple-query|(Q1) Ein normaler Hash-Join|SELECT o.customerid, o.orderid, o.orderdate, ol.orderlineid, ol.quantity FROM orders o INNER JOIN orderlines ol ON ol.orderid = o.orderid;||orders(orderid,customerid,orderdate)@orderlines(orderid,orderlineid,quantity)"
-"eo1|nested-loop-join|(Q2) Nested Loop Join|SELECT o.customerid, o.orderid, o.orderdate, ol.orderlineid, ol.quantity FROM orders o INNER JOIN orderlines ol ON ol.orderid=o.orderid WHERE o.orderid BETWEEN 10 AND 20;||orders(orderid,customerid,orderdate)@orderlines(orderid,orderlineid,quantity)"
-"eo1|non-equi-join|(Q3) Non-Equi Join|SELECT o1.orderid, o2.orderid FROM orders o1 JOIN orders o2 ON o1.orderdate!=o2.orderdate WHERE o1.customerid<100;||orders(customerid)@orders(customerid,orderdate)"
-"eo1|grouped-join|(Q4a)|SELECT cat.categoryname, count(prod.prod_id) anzahl FROM products prod INNER JOIN categories cat ON prod.category=cat.category GROUP BY cat.categoryname ORDER BY cat.categoryname;||products(category)"
-"eo1|grouped-join-reduced|(Q4b) Eingeschränkt|SELECT cat.categoryname, count(prod.prod_id) anzahl FROM products prod INNER JOIN categories cat ON prod.category=cat.category WHERE cat.category = 6 GROUP BY cat.categoryname ORDER BY cat.categoryname;||"
-"eo1|cat11-exists|(Q5) Ein Produkt der Kategorie 11 mit EXISTS|SELECT * FROM orders o WHERE EXISTS ( SELECT * FROM orderlines ol inner join products p on ol.prod_id=p.prod_id WHERE ol.orderId=o.orderid AND P.category=11);||products(category)@orderlines(prod_id)"
-"eo1|cat11-distinct|(Q6) Ein Produkt der Kategorie 11 mit DISTINCT|SELECT DISTINCT o.* FROM orders o inner join orderlines ol on ol.orderId=o.orderid inner join products p on ol.prod_id=p.prod_id WHERE P.category=11;||products(category)@orderlines(prod_id)"
-"eo1|cat11-having|(Q7) Ein Produkt der Kategorie 11 mit HAVING|SELECT o.orderid,o.orderdate,o.customerid,o.netamount,o.tax,o.totalamount FROM orders o inner join orderlines ol on ol.orderId=o.orderid inner join products p on ol.prod_id=p.prod_id GROUP BY p.category,o.orderid,o.orderdate,o.customerid,o.netamount,o.tax,o.totalamount HAVING p.category=11;||products(category)@orderlines(prod_id)"
-"eo1|cat11-cte-join|(A5c1)|with orderlines_to_products_eleven as (select * from products p inner join orderlines ol on ol.prod_id=p.prod_id where p.category=11) select * from orders o inner join orderlines_to_products_eleven ope on o.orderid=ope.orderid;||products(category)@orderlines(prod_id)"
-"eo1|cat11-cte-subselect|(A5c2)|with cat_eleven_products as ( select * from products where category=11 ), orderlines_to_products_eleven as ( select * from orderlines ol, cat_eleven_products cep where ol.prod_id=cep.prod_id ) select * from orders o inner join orderlines_to_products_eleven ope on o.orderid=ope.orderid;||products(category)@orderlines(prod_id)"
+"eo1|star-products-titlelike|(Q23)|select * from products where title like '%ACADEMY FLY%';||products(title)@products using gist(title gist_trgm_ops)"
+"eo1|simple-query|(Q24) Ein normaler Hash-Join|SELECT o.customerid, o.orderid, o.orderdate, ol.orderlineid, ol.quantity FROM orders o INNER JOIN orderlines ol ON ol.orderid = o.orderid;||orders(orderid,customerid,orderdate)@orderlines(orderid,orderlineid,quantity)"
+"eo1|nested-loop-join|(Q25) Nested Loop Join|SELECT o.customerid, o.orderid, o.orderdate, ol.orderlineid, ol.quantity FROM orders o INNER JOIN orderlines ol ON ol.orderid=o.orderid WHERE o.orderid BETWEEN 10 AND 20;||orders(orderid,customerid,orderdate)@orderlines(orderid,orderlineid,quantity)"
+"eo1|non-equi-join|(Q26) Non-Equi Join|SELECT o1.orderid, o2.orderid FROM orders o1 JOIN orders o2 ON o1.orderdate<>o2.orderdate WHERE o1.customerid<100;||orders(customerid)@orders(customerid,orderdate)"
+"eo1|grouped-join|(Q27)|SELECT cat.categoryname, count(prod.prod_id) anzahl FROM products prod INNER JOIN categories cat ON prod.category=cat.category GROUP BY cat.categoryname ORDER BY cat.categoryname;||products(category)"
+"eo1|grouped-join-reduced|(Q28) Eingeschränkt|SELECT cat.categoryname, count(prod.prod_id) anzahl FROM products prod INNER JOIN categories cat ON prod.category=cat.category WHERE cat.category = 6 GROUP BY cat.categoryname ORDER BY cat.categoryname;||"
+"eo1|cat11-exists|(Q29) Ein Produkt der Kategorie 11 mit EXISTS|SELECT * FROM orders o WHERE EXISTS ( SELECT * FROM orderlines ol inner join products p on ol.prod_id=p.prod_id WHERE ol.orderId=o.orderid AND P.category=11);||products(category)@orderlines(prod_id)"
+"eo1|cat11-distinct|(Q30) Ein Produkt der Kategorie 11 mit DISTINCT|SELECT DISTINCT o.* FROM orders o inner join orderlines ol on ol.orderId=o.orderid inner join products p on ol.prod_id=p.prod_id WHERE P.category=11;||products(category)@orderlines(prod_id)"
+"eo1|cat11-having|(Q31) Ein Produkt der Kategorie 11 mit HAVING|SELECT o.orderid,o.orderdate,o.customerid,o.netamount,o.tax,o.totalamount FROM orders o inner join orderlines ol on ol.orderId=o.orderid inner join products p on ol.prod_id=p.prod_id GROUP BY p.category,o.orderid,o.orderdate,o.customerid,o.netamount,o.tax,o.totalamount HAVING p.category=11;||products(category)@orderlines(prod_id)"
+"eo1|cat11-cte-join|(Q32)|with orderlines_to_products_eleven as (select * from products p inner join orderlines ol on ol.prod_id=p.prod_id where p.category=11) select * from orders o inner join orderlines_to_products_eleven ope on o.orderid=ope.orderid;||products(category)@orderlines(prod_id)"
+"eo1|cat11-cte-subselect|(Q33)|with cat_eleven_products as ( select * from products where category=11 ), orderlines_to_products_eleven as ( select * from orderlines ol, cat_eleven_products cep where ol.prod_id=cep.prod_id ) select * from orders o inner join orderlines_to_products_eleven ope on o.orderid=ope.orderid;||products(category)@orderlines(prod_id)"
 )

ColumnStore/execqueries.sh

 #!/bin/sh
 
-_MY_DIR="$(cd $(dirname ${0:-.}) && pwd)"
-_EX_DIR="$(realpath --relative-to=. $_MY_DIR)"
+_MY_DIR="$(cd "$(dirname "${0:-.}")" && pwd)"
+_EX_DIR="$(realpath --relative-to=. "$_MY_DIR")"
+_EX_NAME="$(basename "${_MY_DIR}")"
 test -z "$_EX_DIR" && _EX_DIR=.
-_HELPERS_DIR="$(realpath -e --relative-to=. ${_MY_DIR}/../Helpers)"
+_HELPERS_DIR="$(realpath -e --relative-to=. "${_MY_DIR}/../Helpers")"
+_PARENT_DIR="$(cd "${_MY_DIR}/.." && pwd)"
+_WAIT_FOR_IT_LOCATION=${_PARENT_DIR}/wait-for-it.sh
+_RM_LIST=""
 
+_setup_execqueries() {
+    test -f "${_WAIT_FOR_IT_LOCATION}" || {
+        wget --quiet -O "${_WAIT_FOR_IT_LOCATION}" https://gitlab.dev.ifs.hsr.ch/ifs/wait-for-it/raw/master/wait-for-it.sh ;
+        chmod +x "${_WAIT_FOR_IT_LOCATION}";
+    }
+    test -f "${_WAIT_FOR_IT_LOCATION}" && _RM_LIST="$_RM_LIST:$_WAIT_FOR_IT_LOCATION"
+}
+
+_exitproc() {
+    IFS=":"
+    for f in $_RM_LIST; do
+        rm -f "$f"
+    done
+}
+
+trap _exitproc INT HUP TERM
+
+db_host=postgres
 db_name=${1:-ds2}
 db_user=${2:-$db_name}
-db_pass=${3:-$(printf "$db_user\\n")}
+db_pass=${3:-$db_user}
 docname=${4:-Queries}
+db_data_path="/src/Databases/dvdstore_500mb"
+db_script_PostgreSQL="${db_data_path}/0_runAllScripts.sql"
+
+_load_db_PostgreSQL() {
+    _passfilename="$1";
+    _passfilepath="$2";
+    _compose_file="$3";
+    _root_dir="${4:-.}";
+    _host="$5"
+    _name="$6"
+    _user="$7"
+
+    docker-compose --file "$_compose_file" \
+        --project-name "$_EX_NAME" \
+        --project-directory "$_root_dir" \
+        up -d \
+            postgres-dflt
+
+    printf "\\n\\n" | docker-compose --file "$_compose_file" \
+        --project-name "$_EX_NAME" \
+        --project-directory "$_root_dir" \
+        run \
+            -e "PGPASSFILE=/$_passfilename" -v "${_passfilepath}:/${_passfilename}:ro" \
+            psql-dflt \
+        wait-for-it.sh --timeout=20 --strict \
+            --host=postgres --port=5432 -- \
+        su -c "psql -h \"$_host\" -U \"postgres\" -d \"postgres\" \
+            --no-readline \
+            -f \"$db_script_PostgreSQL\""
+}
+
+_run_queries_PostgreSQL() {
+    _logfile="$1";
+    _queries_file="$2";
+    _passfilename="$3";
+    _passfilepath="$4";
+    _compose_file="$5";
+    _root_dir="${6:-.}";
+    _host="$7"
+    _name="$8"
+    _user="$9"
+    echo "Logging into $_logfile"
+
+    _cname=$(docker-compose --file "$_compose_file" \
+        --project-name "$_EX_NAME" \
+        --project-directory "$_root_dir" \
+        run -d \
+            -e "PGPASSFILE=/$_passfilename" -v "${_passfilepath}:/${_passfilename}:ro" \
+            psql-dflt \
+        wait-for-it.sh --timeout=20 --strict \
+            --host=postgres --port=5432 -- \
+        su -c "psql -h \"$_host\" -U \"$_user\" -d \"$_name\" \
+            --no-readline \
+            --tuples-only \
+            -f \"/src/$_queries_file\"")
+
+    if docker container wait "$_cname"; then
+        docker container logs "$_cname" >"$_logfile" 2>&1;
+        sed -i -r -e 's/ \(actual[^)]+\)//' \
+            -e '/^(SET|BEGIN;?|ROLLBACK;?|CREATE INDEX)$/ d' "$_logfile";
+    else
+        echo "Failed to wait on removed? PostgreSQL container, did it start?"
+    fi
+}
+
+_load_db_MonetDB() {
+    _passfilename="$1";
+    _passfilepath="$2";
+    _compose_file="$3";
+    _root_dir="${4:-.}";
+    _host="$5"
+    _name="$6"
+    _user="$7"
+
+    docker-compose --file "$_compose_file" \
+        --project-name "$_EX_NAME" \
+        --project-directory "$_root_dir" \
+        up -d \
+            monetdb-dflt
+
+    docker-compose --file "$_compose_file" \
+        --project-name "$_EX_NAME" \
+        --project-directory "$_root_dir" \
+        run \
+            --workdir=$db_data_path \
+            --volume "${_PARENT_DIR}":/my_bin:ro \
+            -e "DOTMONETDBFILE=/src/ColumnStore/.monetdb-monetdb" \
+            mclient-batch-dflt \
+        /my_bin/wait-for-it.sh --timeout=20 --strict \
+            --host=monetdb --port=50000 -- \
+            true;
 
-MonetDB() {
-	local _logfile=$1;
-	local _queries_file=$2;
-	echo "Logging into $_logfile"
-	# exec >$_logfile 2>&1
-	# eval docker-compose --file ${_HELPERS_DIR}/docker-compose.yml \
-	# --project-name dataeng \
-	# --project-directory . \
-	# run mclient-batch \
-	# mclient -u $db_user -d $db_name \
-	# --history --interactive --timer=performance \
-	# /src/$_queries_file \
-	# <<<$db_pass
-}
-
-PostgreSQL() {
-	local _logfile=$1;
-	local _queries_file=$2;
-	echo "Logging into $_logfile"
-	exec >$_logfile 2>&1
-	eval docker-compose --file ${_HELPERS_DIR}/docker-compose.yml \
-	--project-name dataeng \
-	--project-directory . \
-	run postgres \
-	psql -h postgres -U $db_user -d $db_name \
-	-f /src/$_queries_file \
-	--no-readline \
-	--tuples-only <<<$db_pass | \
-	sed -r -e 's/ \(actual[^)]+\)//' \
-	-e '/^(SET|BEGIN;?|ROLLBACK;?|CREATE INDEX)$/ d'
-}
-
-for dbname in MonetDB PostgreSQL; do
-	_queries_generate_script=${_EX_DIR}/${docname}_${dbname}.sh
-	_queries_file=${_EX_DIR}/${docname}_${dbname}.sql
-	sh ${_queries_generate_script} >${_queries_file}
-	_test_date="$(date +%Y%m%d%H%M%S)"
-	_logfile="${_EX_DIR}/$(basename ${_MY_DIR}).${dbname}.${_test_date}.log"
-	eval ${dbname} $_logfile $_queries_file
+    docker-compose --file "$_compose_file" \
+        --project-name "$_EX_NAME" \
+        --project-directory "$_root_dir" \
+        exec monetdb-dflt \
+        sh -c "monetdb status; monetdb destroy -f \"$_name\"; \
+            monetdb create \"$_name\" && monetdb release \"$_name\" && \
+            monetdb start \"$_name\"; monetdb status"
+
+    docker-compose --file "$_compose_file" \
+        --project-name "$_EX_NAME" \
+        --project-directory "$_root_dir" \
+        run \
+            --workdir=$db_data_path \
+            --volume "${_PARENT_DIR}":/my_bin:ro \
+            -e "DOTMONETDBFILE=/src/ColumnStore/.monetdb-monetdb" \
+            mclient-batch-dflt \
+        /my_bin/wait-for-it.sh --timeout=20 --strict \
+            --host=monetdb --port=50000 -- \
+        mclient -h monetdb -d "$_name" \
+            --history --interactive \
+            --encoding="UTF-8" \
+            2_schema.monetdb.sql 3_inserts.monetdb.sql 4_keys.monetdb.sql
+
+    docker-compose --file "$_compose_file" \
+        --project-name "$_EX_NAME" \
+        --project-directory "$_root_dir" \
+        run \
+            --workdir=$db_data_path \
+            --volume "${_PARENT_DIR}":/my_bin:ro \
+            -e "DOTMONETDBFILE=/src/ColumnStore/.monetdb-monetdb" \
+            mclient-batch-dflt \
+        /my_bin/wait-for-it.sh --timeout=20 --strict \
+            --host=monetdb --port=50000 -- \
+        mclient -h monetdb -d "$_name" \
+            --history \
+            -s "ANALYZE $_name; select * from sys.statistics; grant sysadmin to $_user with admin option;"
+    #FIXME: set correct permissions to select from sys.tracelog as ds2 user
+}
+
+_run_queries_MonetDB() {
+    _logfile="$1";
+    _queries_file="$2";
+    _passfilename="$3";
+    _passfilepath="$4";
+    _compose_file="$5";
+    _root_dir="${6:-.}";
+    echo "Logging into $_logfile"
+
+    _cname=$(docker-compose --file "$_compose_file" \
+        --project-name "$_EX_NAME" \
+        --project-directory "$_root_dir" \
+        run -d \
+            -e "DOTMONETDBFILE=/$_passfilename" -v "${_passfilepath}:/${_passfilename}:ro" \
+            --volume "${_PARENT_DIR}":/my_bin:ro \
+            mclient-batch-dflt \
+        /my_bin/wait-for-it.sh --timeout=20 --strict \
+            --host=monetdb --port=50000 -- \
+        mclient -h monetdb \
+            --history --interactive \
+            --encoding="UTF-8" \
+            "/src/$_queries_file")
+
+    if docker container wait "$_cname"; then
+        docker container logs "$_cname" >"$_logfile" 2>&1;
+        sed -i -r -e '/^%.*$/ d' -e 's|\r$||' "$_logfile";
+    else
+        echo "Failed to wait on removed? PostgreSQL container, did it start?"
+    fi
+}
+
+_create_passfile_PostgreSQL() {
+    _passfilepath="$1";
+    _host="$2"
+    _name="$3"
+    _user="$4"
+    _pass="$5"
+    # see https://wiki.postgresql.org/wiki/Pgpass and https://www.postgresql.org/docs/current/libpq-pgpass.html
+    # hostname:port:database:username:password
+    printf "%s:%s:%s:%s:%s\n" "$_host" "*" "*" "postgres" "postgres" >"$_passfilepath"
+    printf "%s:%s:%s:%s:%s\n" "$_host" "*" "$_name" "$_user" "$_pass" >>"$_passfilepath"
+    chmod 0600 "$_passfilepath"
+}
+
+_create_passfile_MonetDB() {
+    _passfilepath="$1";
+    _host="$2"
+    _name="$3"
+    _user="$4"
+    _pass="$5"
+    # see https://www.monetdb.org/Documentation/mclient-man-page
+    printf "database=%s\nuser=%s\npassword=%s\n" "$_name" "$_user" "$_pass" >"$_passfilepath"
+    chmod 0600 "$_passfilepath"
+}
+
+_setup_execqueries
+for dbtype in MonetDB PostgreSQL; do
+    _queries_generate_script="${_EX_DIR}/${docname}_${dbtype}.sh"
+    test -f "$_queries_generate_script" || continue
+    _queries_file=${_EX_DIR}/${docname}_${dbtype}.sql
+    _test_date="$(date +%Y%m%d%H%M%S)"
+    _logbase="${_EX_DIR}/${_EX_NAME}.${dbtype}"
+    _logfile="${_logbase}.${_test_date}.log"
+
+    sh "${_queries_generate_script}" "$db_name" "$db_user" >"$_queries_file"
+
+    _passfilename=".passfile.$dbtype"
+    _passfilepath="${_MY_DIR}/$_passfilename"
+    _create_passfile_${dbtype} "$_passfilepath" "$db_host" "$db_name" "$db_user" "$db_pass"
+    _load_db_$dbtype "$_passfilename" "$_passfilepath" "${_HELPERS_DIR}/docker-compose.yml" "$_PARENT_DIR" "$db_host" "$db_name" "$db_user"
+    _run_queries_${dbtype} "$_logfile" "$_queries_file" "$_passfilename" "$_passfilepath" "${_HELPERS_DIR}/docker-compose.yml" "$_PARENT_DIR" "$db_host" "$db_name" "$db_user"
+    cp -fp "$_logfile" "${_logbase}.log"
+    _RM_LIST="$_RM_LIST:$_passfilepath"
 done
+_exitproc
+
+echo "Terminate services with this command:"
+echo "docker-compose --file ${_HELPERS_DIR}/docker-compose.yml --project-name $_EX_NAME --project-directory $_PARENT_DIR down --remove-orphans"

Helpers/docker-compose.yml

@@ -7,6 +7,10 @@ networks:
     attachable: true
     driver: overlay
     name: mongonet
+  monetnet:
+    attachable: true
+    driver: overlay
+    name: monetnet
   neonet:
     attachable: true
     driver: overlay
@@ -363,31 +367,104 @@ services:
     volumes:
     - .:/src:rw
 # end::neoshell[]
+  monetdef: &monet_default
+    environment:
+      <<: *dflt-env
+    image: monetdb/monetdb-r-docker
+    networks:
+      default: null
+# tag::monetdb-dflt[]
+  monetdb-dflt:
+    <<: *monet_default
+    hostname: monetdb
+    networks:
+      default:
+        aliases:
+          - monetdb
+    volumes:
+    - monetdb-data:/var/monetdb5:rw
+    - .:/src:ro
+# end::monetdb-dflt[]
+# tag::mclient-dflt[]
+  mclient-dflt:
+    <<: *monet_default
+    command:
+    - mclient
+    - -h
+    - monetdb
+    - -d
+    - db
+    - --history
+    - --timer=performance
+    depends_on:
+      - monetdb-dflt
+    environment:
+      <<: *dflt-env
+      COLUMNS: '300'
+      #DOTMONETDBFILE: /src/ColumnStore/.monetdb-ds2
+      DOTMONETDBFILE: /src/ColumnStore/.monetdb-monetdb
+    hostname: mclient
+    networks:
+      default: {}
+    stdin_open: true
+    tty: true
+    volumes:
+    - mclient-data:/root:rw
+    - .:/src:rw
+# end::mclient-dflt[]
+# tag::mclient-batch-dflt[]
+  mclient-batch-dflt:
+    <<: *monet_default
+    command:
+    - mclient
+    - -h
+    - monetdb
+    - -d
+    - db
+    depends_on:
+      - monetdb-dflt
+    environment:
+      <<: *dflt-env
+      #DOTMONETDBFILE: /src/ColumnStore/.monetdb-ds2
+      DOTMONETDBFILE: /src/ColumnStore/.monetdb-monetdb
+    hostname: mclient
+    networks:
+      default: {}
+    stdin_open: true
+    volumes:
+    - mclient-data:/root:rw
+    - .:/src:rw
+# end::mclient-batch-dflt[]
 # tag::monetdb[]
   monetdb:
+    <<: *monet_default
     container_name: monetdb-r
-    image: monetdb/monetdb-r-docker
-    ports:
-    - published: 50000
-      target: 50000
+    hostname: monetdb
+    networks:
+      monetnet: null
     volumes:
     - monetdb-data:/var/monetdb5:rw
     - .:/src:ro
 # end::monetdb[]
 # tag::mclient[]
   mclient:
+    <<: *monet_default
     command:
     - mclient
-    - -u
+    - -h
     - monetdb
     - -d
     - db
     - --history
     - --timer=performance
     environment:
+      <<: *dflt-env
       COLUMNS: '300'
-    image: monetdb/monetdb-r-docker
-    network_mode: host
+      #DOTMONETDBFILE: /src/ColumnStore/.monetdb-ds2
+      DOTMONETDBFILE: /src/ColumnStore/.monetdb-monetdb
+    hostname: mclient
+    networks:
+      monetnet: null
     stdin_open: true
     tty: true
     volumes:
@@ -396,14 +473,21 @@ services:
 # end::mclient[]
 # tag::mclient-batch[]
   mclient-batch:
+    <<: *monet_default
     command:
     - mclient
-    - -u
+    - -h
     - monetdb
     - -d
     - db
-    image: monetdb/monetdb-r-docker
-    network_mode: host
+    environment:
+      <<: *dflt-env
+      #DOTMONETDBFILE: /src/ColumnStore/.monetdb-ds2
+      DOTMONETDBFILE: /src/ColumnStore/.monetdb-monetdb
+      USER: monetdb
+    hostname: mclient
+    networks:
+      monetnet: null
     stdin_open: true
     volumes:
     - mclient-data:/root:rw