Skip to content

GitLab

Commit 8bf37c8f authored by Marcel Huber's avatar Marcel Huber
Browse files

more shellcheck based cleanup

parent c9866fba
Hide whitespace changes
Inline Side-by-side
ColumnStore/Queries_MonetDB.sh
View file @ 8bf37c8f
......@@ -4,9 +4,6 @@ _MY_DIR="$(cd "$(dirname "${0:-.}")" && pwd)"
_EX_DIR="$(realpath --relative-to=. "$_MY_DIR")"
test -z "$_EX_DIR" && _EX_DIR=.
eo0=""
eo1=""
# shellcheck source=./Queries_common.sh
. $_EX_DIR/Queries_common.sh
# disable globbing for star queries (*)
......@@ -24,36 +21,37 @@ set schema ds2;
set role sysadmin;
--
$(for q in "${queries[@]}"; do
explainopts="$(eval echo $(echo \"\$$(cut -d'|' -f1 <<<"$q")\"))";
aufgabe="$(cut -d'|' -f2 <<<"$q")";
aufgabe_desc="$(cut -d'|' -f3 <<<"$q")";
query="$(cut -d'|' -f4 <<<"$q")";
query_escaped="$(echo -n "$query" | sed 's/\x27/\\\x27/g')";
echo "\f tab"
echo "select '--== Query $aufgabe_desc ==--';"
echo "select '-- tag::${aufgabe}[]';"
echo "select '-- tag::${aufgabe}-query[]';"
echo "select '$query_escaped';"
echo "select '-- end::${aufgabe}-query[]';"
echo "select '-- tag::${aufgabe}-plan[]';"
echo "\f raw"
echo "plan $query"
echo "\f tab"
echo "select '-- end::${aufgabe}-plan[]';"
echo "select '-- tag::${aufgabe}-timing[]';"
echo "\f trash"
echo "\t performance"
echo "$query"
echo "\t none"
echo "\f tab"
echo "select '-- end::${aufgabe}-timing[]';"
echo "select '-- tag::${aufgabe}-trace[]';"
echo "\f rowcount"
echo "trace $query"
echo "\f csv+;"
echo "select ticks as usec, stmt from sys.tracelog;"
echo "\f tab"
echo "select '-- end::${aufgabe}-trace[]';"
echo "select '-- end::${aufgabe}[]';"
aufgabe="$(cut -d'|' -f2 <<<"$q")";
aufgabe_desc="$(cut -d'|' -f3 <<<"$q")";
query="$(cut -d'|' -f4 <<<"$q")";
query_escaped="$(echo -n "$query" | sed 's/\x27/\\\x27/g')";
echo "\f tab"
echo "select '--== Query $aufgabe_desc ==--';"
echo "select '-- tag::${aufgabe}[]';"
echo "select '-- tag::${aufgabe}-query[]';"
echo "select '$query_escaped';"
echo "select '-- end::${aufgabe}-query[]';"
echo "select '-- tag::${aufgabe}-plan[]';"
echo "\f raw"
echo "plan $query"
echo "\f tab"
echo "select '-- end::${aufgabe}-plan[]';"
echo "select '-- tag::${aufgabe}-timing[]';"
echo "\f trash"
# shellcheck disable=SC2028
echo "\t performance"
echo "$query"
# shellcheck disable=SC2028
echo "\t none"
echo "\f tab"
echo "select '-- end::${aufgabe}-timing[]';"
echo "select '-- tag::${aufgabe}-trace[]';"
echo "\f rowcount"
echo "trace $query"
echo "\f csv+;"
echo "select ticks as usec, stmt from sys.tracelog;"
echo "\f tab"
echo "select '-- end::${aufgabe}-trace[]';"
echo "select '-- end::${aufgabe}[]';"
done)
EOF
ColumnStore/Queries_PostgreSQL.sh
View file @ 8bf37c8f
......@@ -6,8 +6,9 @@ test -z "$_EX_DIR" && _EX_DIR=.
db_name=${1:-ds2}
db_user=${2:-$db_name}
# shellcheck disable=SC2034
eo0=""
# shellcheck disable=SC2034
eo1="EXPLAIN (ANALYZE 1,COSTS 0,TIMING 1,BUFFERS 0,VERBOSE 0,FORMAT TEXT)"
# shellcheck source=./Queries_common.sh
......@@ -41,30 +42,30 @@ VACUUM (ANALYZE);
\c $db_name $db_user
$(for q in "${queries[@]}"; do
explainopts="$(eval echo $(echo \"\$$(cut -d'|' -f1 <<<"$q")\"))";
aufgabe="$(cut -d'|' -f2 <<<"$q")";
aufgabe_desc="$(cut -d'|' -f3 <<<"$q")";
query="$(cut -d'|' -f4 <<<"$q")";
echo "${echostr}--== Query $aufgabe_desc ==--"
echo "${echostr}-- tag::${aufgabe}[]"
echo "\! echo \"$explainopts\" >/tmp/qa"
echo "${echostr}-- tag::${aufgabe}-eo[]"
echo "\! cat /tmp/qa"
echo "${echostr}-- end::${aufgabe}-eo[]"
echo "\! echo \"$query\" | pg_format -f1 -s2 -u2 | tee -a /tmp/qa >/tmp/q"
echo "${echostr}-- tag::${aufgabe}-query[]"
echo "\! cat /tmp/q"
echo "${echostr}-- end::${aufgabe}-query[]"
echo "${echostr}-- tag::${aufgabe}-explain[]"
echo "\pset format asciidoc"
echo "${echostr}-- tag::${aufgabe}-explain-adoc[]"
echo "\i /tmp/qa"
echo "${echostr}-- end::${aufgabe}-explain-adoc[]"
echo "\pset format ${format_text}"
echo "${echostr}-- tag::${aufgabe}-explain-text[]"
echo "\i /tmp/qa"
echo "${echostr}-- end::${aufgabe}-explain-text[]"
echo "${echostr}-- end::${aufgabe}-explain[]"
echo "${echostr}-- end::${aufgabe}[]"
explainopts="$(eval echo \$"$(cut -d'|' -f1 <<<"$q")")";
aufgabe="$(cut -d'|' -f2 <<<"$q")";
aufgabe_desc="$(cut -d'|' -f3 <<<"$q")";
query="$(cut -d'|' -f4 <<<"$q")";
echo "${echostr}--== Query $aufgabe_desc ==--"
echo "${echostr}-- tag::${aufgabe}[]"
echo "\! echo \"$explainopts\" >/tmp/qa"
echo "${echostr}-- tag::${aufgabe}-eo[]"
echo "\! cat /tmp/qa"
echo "${echostr}-- end::${aufgabe}-eo[]"
echo "\! echo \"$query\" | pg_format -f1 -s2 -u2 | tee -a /tmp/qa >/tmp/q"
echo "${echostr}-- tag::${aufgabe}-query[]"
echo "\! cat /tmp/q"
echo "${echostr}-- end::${aufgabe}-query[]"
echo "${echostr}-- tag::${aufgabe}-explain[]"
echo "\pset format asciidoc"
echo "${echostr}-- tag::${aufgabe}-explain-adoc[]"
echo "\i /tmp/qa"
echo "${echostr}-- end::${aufgabe}-explain-adoc[]"
echo "\pset format ${format_text}"
echo "${echostr}-- tag::${aufgabe}-explain-text[]"
echo "\i /tmp/qa"
echo "${echostr}-- end::${aufgabe}-explain-text[]"
echo "${echostr}-- end::${aufgabe}-explain[]"
echo "${echostr}-- end::${aufgabe}[]"
done)
EOF
ColumnStore/execqueries.sh
View file @ 8bf37c8f
......@@ -10,18 +10,18 @@ _WAIT_FOR_IT_LOCATION=${_PARENT_DIR}/wait-for-it.sh
_RM_LIST=""
_setup_execqueries() {
test -f "${_WAIT_FOR_IT_LOCATION}" || {
wget --quiet -O "${_WAIT_FOR_IT_LOCATION}" https://gitlab.dev.ifs.hsr.ch/ifs/wait-for-it/raw/master/wait-for-it.sh ;
chmod +x "${_WAIT_FOR_IT_LOCATION}";
}
test -f "${_WAIT_FOR_IT_LOCATION}" && _RM_LIST="$_RM_LIST:$_WAIT_FOR_IT_LOCATION"
test -f "${_WAIT_FOR_IT_LOCATION}" || {
wget --quiet -O "${_WAIT_FOR_IT_LOCATION}" https://gitlab.dev.ifs.hsr.ch/ifs/wait-for-it/raw/master/wait-for-it.sh ;
chmod +x "${_WAIT_FOR_IT_LOCATION}";
}
test -f "${_WAIT_FOR_IT_LOCATION}" && _RM_LIST="$_RM_LIST:$_WAIT_FOR_IT_LOCATION"
}
_exitproc() {
IFS=":"
for f in $_RM_LIST; do
rm -f "$f"
done
IFS=":"
for f in $_RM_LIST; do
rm -f "$f"
done
}
trap _exitproc INT HUP TERM
......@@ -35,206 +35,206 @@ db_data_path="/src/Databases/dvdstore_500mb"
db_script_PostgreSQL="${db_data_path}/0_runAllScripts.sql"
_load_db_PostgreSQL() {
_passfilename="$1";
_passfilepath="$2";
_compose_file="$3";
_root_dir="${4:-.}";
_host="$5"
_name="$6"
_user="$7"
docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
up -d \
postgres-dflt
printf "\\n\\n" | docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
run \
-e "PGPASSFILE=/$_passfilename" -v "${_passfilepath}:/${_passfilename}:ro" \
psql-dflt \
wait-for-it.sh --timeout=20 --strict \
--host=postgres --port=5432 -- \
su -c "psql -h \"$_host\" -U \"postgres\" -d \"postgres\" \
--no-readline \
-f \"$db_script_PostgreSQL\""
_passfilename="$1";
_passfilepath="$2";
_compose_file="$3";
_root_dir="${4:-.}";
_host="$5"
_name="$6"
_user="$7"
docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
up -d \
postgres-dflt
printf "\\n\\n" | docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
run \
-e "PGPASSFILE=/$_passfilename" -v "${_passfilepath}:/${_passfilename}:ro" \
psql-dflt \
wait-for-it.sh --timeout=20 --strict \
--host=postgres --port=5432 -- \
su -c "psql -h \"$_host\" -U \"postgres\" -d \"postgres\" \
--no-readline \
-f \"$db_script_PostgreSQL\""
}
_run_queries_PostgreSQL() {
_logfile="$1";
_queries_file="$2";
_passfilename="$3";
_passfilepath="$4";
_compose_file="$5";
_root_dir="${6:-.}";
_host="$7"
_name="$8"
_user="$9"
echo "Logging into $_logfile"
_cname=$(docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
run -d \
-e "PGPASSFILE=/$_passfilename" -v "${_passfilepath}:/${_passfilename}:ro" \
psql-dflt \
wait-for-it.sh --timeout=20 --strict \
--host=postgres --port=5432 -- \
su -c "psql -h \"$_host\" -U \"$_user\" -d \"$_name\" \
--no-readline \
--tuples-only \
-f \"/src/$_queries_file\"")
if docker container wait "$_cname"; then
docker container logs "$_cname" >"$_logfile" 2>&1;
sed -i -r -e 's/ \(actual[^)]+\)//' \
-e '/^(SET|BEGIN;?|ROLLBACK;?|CREATE INDEX)$/ d' "$_logfile";
else
echo "Failed to wait on removed? PostgreSQL container, did it start?"
fi
_logfile="$1";
_queries_file="$2";
_passfilename="$3";
_passfilepath="$4";
_compose_file="$5";
_root_dir="${6:-.}";
_host="$7"
_name="$8"
_user="$9"
echo "Logging into $_logfile"
_cname=$(docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
run -d \
-e "PGPASSFILE=/$_passfilename" -v "${_passfilepath}:/${_passfilename}:ro" \
psql-dflt \
wait-for-it.sh --timeout=20 --strict \
--host=postgres --port=5432 -- \
su -c "psql -h \"$_host\" -U \"$_user\" -d \"$_name\" \
--no-readline \
--tuples-only \
-f \"/src/$_queries_file\"")
if docker container wait "$_cname"; then
docker container logs "$_cname" >"$_logfile" 2>&1;
sed -i -r -e 's/ \(actual[^)]+\)//' \
-e '/^(SET|BEGIN;?|ROLLBACK;?|CREATE INDEX)$/ d' "$_logfile";
else
echo "Failed to wait on removed? PostgreSQL container, did it start?"
fi
}
_load_db_MonetDB() {
_passfilename="$1";
_passfilepath="$2";
_compose_file="$3";
_root_dir="${4:-.}";
_host="$5"
_name="$6"
_user="$7"
docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
up -d \
monetdb-dflt
docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
run \
--workdir=$db_data_path \
--volume "${_PARENT_DIR}":/my_bin:ro \
-e "DOTMONETDBFILE=/src/ColumnStore/.monetdb-monetdb" \
mclient-batch-dflt \
/my_bin/wait-for-it.sh --timeout=20 --strict \
--host=monetdb --port=50000 -- \
true;
docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
exec monetdb-dflt \
sh -c "monetdb status; monetdb destroy -f \"$_name\"; \
monetdb create \"$_name\" && monetdb release \"$_name\" && \
monetdb start \"$_name\"; monetdb status"
docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
run \
--workdir=$db_data_path \
--volume "${_PARENT_DIR}":/my_bin:ro \
-e "DOTMONETDBFILE=/src/ColumnStore/.monetdb-monetdb" \
mclient-batch-dflt \
/my_bin/wait-for-it.sh --timeout=20 --strict \
--host=monetdb --port=50000 -- \
mclient -h monetdb -d "$_name" \
--history --interactive \
--encoding="UTF-8" \
2_schema.monetdb.sql 3_inserts.monetdb.sql 4_keys.monetdb.sql
docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
run \
--workdir=$db_data_path \
--volume "${_PARENT_DIR}":/my_bin:ro \
-e "DOTMONETDBFILE=/src/ColumnStore/.monetdb-monetdb" \
mclient-batch-dflt \
/my_bin/wait-for-it.sh --timeout=20 --strict \
--host=monetdb --port=50000 -- \
mclient -h monetdb -d "$_name" \
--history \
-s "ANALYZE $_name; select * from sys.statistics; grant sysadmin to $_user with admin option;"
#FIXME: set correct permissions to select from sys.tracelog as ds2 user
_passfilename="$1";
_passfilepath="$2";
_compose_file="$3";
_root_dir="${4:-.}";
_host="$5"
_name="$6"
_user="$7"
docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
up -d \
monetdb-dflt
docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
run \
--workdir=$db_data_path \
--volume "${_PARENT_DIR}":/my_bin:ro \
-e "DOTMONETDBFILE=/src/ColumnStore/.monetdb-monetdb" \
mclient-batch-dflt \
/my_bin/wait-for-it.sh --timeout=20 --strict \
--host=monetdb --port=50000 -- \
true;
docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
exec monetdb-dflt \
sh -c "monetdb status; monetdb destroy -f \"$_name\"; \
monetdb create \"$_name\" && monetdb release \"$_name\" && \
monetdb start \"$_name\"; monetdb status"
docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
run \
--workdir=$db_data_path \
--volume "${_PARENT_DIR}":/my_bin:ro \
-e "DOTMONETDBFILE=/src/ColumnStore/.monetdb-monetdb" \
mclient-batch-dflt \
/my_bin/wait-for-it.sh --timeout=20 --strict \
--host=monetdb --port=50000 -- \
mclient -h monetdb -d "$_name" \
--history --interactive \
--encoding="UTF-8" \
2_schema.monetdb.sql 3_inserts.monetdb.sql 4_keys.monetdb.sql
docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
run \
--workdir=$db_data_path \
--volume "${_PARENT_DIR}":/my_bin:ro \
-e "DOTMONETDBFILE=/src/ColumnStore/.monetdb-monetdb" \
mclient-batch-dflt \
/my_bin/wait-for-it.sh --timeout=20 --strict \
--host=monetdb --port=50000 -- \
mclient -h monetdb -d "$_name" \
--history \
-s "ANALYZE $_name; select * from sys.statistics; grant sysadmin to $_user with admin option;"
#FIXME: set correct permissions to select from sys.tracelog as ds2 user
}
_run_queries_MonetDB() {
_logfile="$1";
_queries_file="$2";
_passfilename="$3";
_passfilepath="$4";
_compose_file="$5";
_root_dir="${6:-.}";
echo "Logging into $_logfile"
_cname=$(docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
run -d \
-e "DOTMONETDBFILE=/$_passfilename" -v "${_passfilepath}:/${_passfilename}:ro" \
--volume "${_PARENT_DIR}":/my_bin:ro \
mclient-batch-dflt \
/my_bin/wait-for-it.sh --timeout=20 --strict \
--host=monetdb --port=50000 -- \
mclient -h monetdb \
--history --interactive \
--encoding="UTF-8" \
"/src/$_queries_file")
if docker container wait "$_cname"; then
docker container logs "$_cname" >"$_logfile" 2>&1;
sed -i -r -e '/^%.*$/ d' -e 's|\r$||' "$_logfile";
else
echo "Failed to wait on removed? PostgreSQL container, did it start?"
fi
_logfile="$1";
_queries_file="$2";
_passfilename="$3";
_passfilepath="$4";
_compose_file="$5";
_root_dir="${6:-.}";
echo "Logging into $_logfile"
_cname=$(docker-compose --file "$_compose_file" \
--project-name "$_EX_NAME" \
--project-directory "$_root_dir" \
run -d \
-e "DOTMONETDBFILE=/$_passfilename" -v "${_passfilepath}:/${_passfilename}:ro" \
--volume "${_PARENT_DIR}":/my_bin:ro \
mclient-batch-dflt \
/my_bin/wait-for-it.sh --timeout=20 --strict \
--host=monetdb --port=50000 -- \
mclient -h monetdb \
--history --interactive \
--encoding="UTF-8" \
"/src/$_queries_file")
if docker container wait "$_cname"; then
docker container logs "$_cname" >"$_logfile" 2>&1;
sed -i -r -e '/^%.*$/ d' -e 's|\r$||' "$_logfile";
else
echo "Failed to wait on removed? PostgreSQL container, did it start?"
fi
}
_create_passfile_PostgreSQL() {
_passfilepath="$1";
_host="$2"
_name="$3"
_user="$4"
_pass="$5"
# see https://wiki.postgresql.org/wiki/Pgpass and https://www.postgresql.org/docs/current/libpq-pgpass.html
# hostname:port:database:username:password
printf "%s:%s:%s:%s:%s\n" "$_host" "*" "*" "postgres" "postgres" >"$_passfilepath"
printf "%s:%s:%s:%s:%s\n" "$_host" "*" "$_name" "$_user" "$_pass" >>"$_passfilepath"
chmod 0600 "$_passfilepath"
_passfilepath="$1";
_host="$2"
_name="$3"
_user="$4"
_pass="$5"
# see https://wiki.postgresql.org/wiki/Pgpass and https://www.postgresql.org/docs/current/libpq-pgpass.html
# hostname:port:database:username:password
printf "%s:%s:%s:%s:%s\n" "$_host" "*" "*" "postgres" "postgres" >"$_passfilepath"
printf "%s:%s:%s:%s:%s\n" "$_host" "*" "$_name" "$_user" "$_pass" >>"$_passfilepath"
chmod 0600 "$_passfilepath"
}
_create_passfile_MonetDB() {
_passfilepath="$1";
_host="$2"
_name="$3"
_user="$4"
_pass="$5"
# see https://www.monetdb.org/Documentation/mclient-man-page
printf "database=%s\nuser=%s\npassword=%s\n" "$_name" "$_user" "$_pass" >"$_passfilepath"
chmod 0600 "$_passfilepath"
_passfilepath="$1";
_host="$2"
_name="$3"
_user="$4"
_pass="$5"
# see https://www.monetdb.org/Documentation/mclient-man-page
printf "database=%s\nuser=%s\npassword=%s\n" "$_name" "$_user" "$_pass" >"$_passfilepath"
chmod 0600 "$_passfilepath"
}
_setup_execqueries
for dbtype in MonetDB PostgreSQL; do
_queries_generate_script="${_EX_DIR}/${docname}_${dbtype}.sh"
test -f "$_queries_generate_script" || continue
_queries_file=${_EX_DIR}/${docname}_${dbtype}.sql
_test_date="$(date +%Y%m%d%H%M%S)"
_logbase="${_EX_DIR}/${_EX_NAME}.${dbtype}"
_logfile="${_logbase}.${_test_date}.log"
sh "${_queries_generate_script}" "$db_name" "$db_user" >"$_queries_file"
_passfilename=".passfile.$dbtype"
_passfilepath="${_MY_DIR}/$_passfilename"
_create_passfile_${dbtype} "$_passfilepath" "$db_host" "$db_name" "$db_user" "$db_pass"
_load_db_$dbtype "$_passfilename" "$_passfilepath" "${_HELPERS_DIR}/docker-compose.yml" "$_PARENT_DIR" "$db_host" "$db_name" "$db_user"
_run_queries_${dbtype} "$_logfile" "$_queries_file" "$_passfilename" "$_passfilepath" "${_HELPERS_DIR}/docker-compose.yml" "$_PARENT_DIR" "$db_host" "$db_name" "$db_user"
cp -fp "$_logfile" "${_logbase}.log"
_RM_LIST="$_RM_LIST:$_passfilepath"
_queries_generate_script="${_EX_DIR}/${docname}_${dbtype}.sh"
test -f "$_queries_generate_script" || continue
_queries_file=${_EX_DIR}/${docname}_${dbtype}.sql
_test_date="$(date +%Y%m%d%H%M%S)"
_logbase="${_EX_DIR}/${_EX_NAME}.${dbtype}"
_logfile="${_logbase}.${_test_date}.log"
sh "${_queries_generate_script}" "$db_name" "$db_user" >"$_queries_file"
_passfilename=".passfile.$dbtype"
_passfilepath="${_MY_DIR}/$_passfilename"
_create_passfile_${dbtype} "$_passfilepath" "$db_host" "$db_name" "$db_user" "$db_pass"
_load_db_$dbtype "$_passfilename" "$_passfilepath" "${_HELPERS_DIR}/docker-compose.yml" "$_PARENT_DIR" "$db_host" "$db_name" "$db_user"
_run_queries_${dbtype} "$_logfile" "$_queries_file" "$_passfilename" "$_passfilepath" "${_HELPERS_DIR}/docker-compose.yml" "$_PARENT_DIR" "$db_host" "$db_name" "$db_user"
cp -fp "$_logfile" "${_logbase}.log"
_RM_LIST="$_RM_LIST:$_passfilepath"
done
_exitproc
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment