cerxes/openmaptiles
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

python3 qa preparations (#711)

Browse Source
This commit is contained in:
Yuri Astrakhan 2019-11-20 12:47:12 -05:00 committed by GitHub
parent 39dcfc2072
commit 8bb3236eda
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 27 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
qa/table_sizes.py
View File

@ -3,10 +3,9 @@ import sys
import argparse import argparse
import subprocess import subprocess
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('--noan', action='store_true', help='Not to run make psql-analyze') parser.add_argument('--noan', action='store_true', help='Not to run make psql-analyze')
TOTAL_SIZE_SQL = """SELECT TOTAL_SIZE_SQL = """SELECT
pg_size_pretty(sum(size)) AS size pg_size_pretty(sum(size)) AS size
FROM ( FROM (
@ -18,8 +17,7 @@ FROM (
) a ) a
;""".replace('\"', '\\\"') ;""".replace('\"', '\\\"')
TABLE_SIZES_SQL = """SELECT
TABLE_SIZES_SQL="""SELECT
a.relname as "table", a.relname as "table",
pg_table_size(a.relid) as "size", pg_table_size(a.relid) as "size",
b.n_live_tup as "rows" b.n_live_tup as "rows"
@ -30,7 +28,6 @@ WHERE
ORDER BY a.relname; ORDER BY a.relname;
""".replace('\"', '\\\"') """.replace('\"', '\\\"')
TABLES_SQL = """SELECT TABLES_SQL = """SELECT
a.relname a.relname
FROM pg_catalog.pg_statio_user_tables a FROM pg_catalog.pg_statio_user_tables a
@ -59,34 +56,38 @@ COLUMNS_SQL = """select
from {1} t; from {1} t;
""".replace('\"', '\\\"') """.replace('\"', '\\\"')
def print_column_sizes(tables): def print_column_sizes(tables):
for table in tables: for table in tables:
print "Column sizes of table "+table print("Column sizes of table " + table)
cmds = [ cmds = [
'docker-compose run --rm import-osm', 'docker-compose run --rm import-osm',
'/usr/src/app/psql.sh -t -A -F\",\" -P pager=off', '/usr/src/app/psql.sh -t -A -F\",\" -P pager=off',
'-c \"' + COLUMN_NAMES_SQL.format(table).replace('\n', ' ').replace('\r', '') + '\"' '-c \"' + COLUMN_NAMES_SQL.format(table).replace('\n', ' ').replace('\r',
'') + '\"'
] ]
# print " ".join(cmds) # print " ".join(cmds)
output = subprocess.check_output(" ".join(cmds), shell=True) output = subprocess.check_output(" ".join(cmds), shell=True)
columns = filter(lambda c: len(c)>0, map(lambda l: l.strip(), output.split('\n'))) columns = filter(lambda c: len(c) > 0,
map(lambda l: l.strip(), output.split('\n')))
# print columns # print columns
col_sql = ",\n".join(map(lambda c: "sum(pg_column_size(\\\""+c+"\\\")) as \\\""+c+"\\\"", columns)) col_sql = ",\n".join(
map(lambda c: "sum(pg_column_size(\\\"" + c + "\\\")) as \\\"" + c + "\\\"",
columns))
# print COLUMNS_SQL.format(col_sql, table); # print COLUMNS_SQL.format(col_sql, table);
cmds = [ cmds = [
'docker-compose run --rm import-osm', 'docker-compose run --rm import-osm',
'/usr/src/app/psql.sh -F\",\" --no-align -P pager=off', '/usr/src/app/psql.sh -F\",\" --no-align -P pager=off',
'-c \"' + COLUMNS_SQL.format(col_sql, table).replace('\n', ' ').replace('\r', '') + '\"' '-c \"' + COLUMNS_SQL.format(col_sql, table).replace('\n', ' ').replace(
'\r', '') + '\"'
] ]
# print " ".join(cmds) # print " ".join(cmds)
col_csv = subprocess.check_output(" ".join(cmds), shell=True) col_csv = subprocess.check_output(" ".join(cmds), shell=True)
print col_csv print(col_csv)
if __name__ == "__main__": if __name__ == "__main__":
@ -94,12 +95,11 @@ if __name__ == "__main__":
try: try:
if(not args.noan): if not args.noan:
print "Running make psql-analyze" print("Running make psql-analyze")
subprocess.check_output("make psql-analyze", shell=True) subprocess.check_output("make psql-analyze", shell=True)
print("Total size of tables")
print "Total size of tables"
cmds = [ cmds = [
'docker-compose run --rm import-osm', 'docker-compose run --rm import-osm',
'/usr/src/app/psql.sh -F\",\" --no-align -P pager=off', '/usr/src/app/psql.sh -F\",\" --no-align -P pager=off',
@ -107,11 +107,10 @@ if __name__ == "__main__":
] ]
# print " ".join(cmds) # print " ".join(cmds)
TOTAL_SIZE_CSV = subprocess.check_output(" ".join(cmds), shell=True) TOTAL_SIZE_CSV = subprocess.check_output(" ".join(cmds), shell=True)
print TOTAL_SIZE_CSV print(TOTAL_SIZE_CSV)
print "\n" print("\n")
print("Table sizes")
print "Table sizes"
cmds = [ cmds = [
'docker-compose run --rm import-osm', 'docker-compose run --rm import-osm',
'/usr/src/app/psql.sh -F\",\" --no-align -P pager=off', '/usr/src/app/psql.sh -F\",\" --no-align -P pager=off',
@ -119,11 +118,10 @@ if __name__ == "__main__":
] ]
# print " ".join(cmds) # print " ".join(cmds)
TABLE_SIZES_CSV = subprocess.check_output(" ".join(cmds), shell=True) TABLE_SIZES_CSV = subprocess.check_output(" ".join(cmds), shell=True)
print TABLE_SIZES_CSV print(TABLE_SIZES_CSV)
print "\n" print("\n")
print("Column sizes")
print "Column sizes"
cmds = [ cmds = [
'docker-compose run --rm import-osm', 'docker-compose run --rm import-osm',
'/usr/src/app/psql.sh -t -A -F\",\" -P pager=off', '/usr/src/app/psql.sh -t -A -F\",\" -P pager=off',
@ -131,11 +129,12 @@ if __name__ == "__main__":
] ]
# print " ".join(cmds) # print " ".join(cmds)
output = subprocess.check_output(" ".join(cmds), shell=True) output = subprocess.check_output(" ".join(cmds), shell=True)
tables = filter(lambda t: len(t)>0, map(lambda l: l.strip(), output.split('\n'))) tables = filter(lambda t: len(t) > 0,
map(lambda l: l.strip(), output.split('\n')))
print_column_sizes(tables); print_column_sizes(tables);
# print tables # print tables
except subprocess.CalledProcessError, e: except subprocess.CalledProcessError as e:
print "Error:\n", e.output print("Error:\n", e.output)
sys.exit(0) sys.exit(0)