diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/ColumnHolder.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/ColumnHolder.java new file mode 100644 index 0000000000000000000000000000000000000000..8f782bbdbae46a4afe6fbc7d562b2169e6ac1e09 --- /dev/null +++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/ColumnHolder.java @@ -0,0 +1,108 @@ +/* + * _____________________________________________________________________________ + * + * INAF - OATS National Institute for Astrophysics - Astronomical Observatory of + * Trieste INAF - IA2 Italian Center for Astronomical Archives + * _____________________________________________________________________________ + * + * Copyright (C) 2017 Istituto Nazionale di Astrofisica + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License Version 3 as published by the + * Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package it.inaf.ia2.tsm; + +import java.io.Serializable; +import java.util.Objects; +import java.util.regex.Pattern; + +/** + * + * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} + */ +public class ColumnHolder implements Serializable { + + private static final long serialVersionUID = 5837270912530393588L; + + private String schemaName; + private String tableName; + private String columnName; + + private ColumnHolder() { + } + + public ColumnHolder(String schemaName, String tableName, String columnName) { + this.schemaName = schemaName; + this.tableName = tableName; + this.columnName = columnName; + } + + public String getSchemaName() { + return schemaName; + } + + public void setSchemaName(String schemaName) { + this.schemaName = schemaName; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getColumnName() { + return columnName; + } + + public void setColumnName(String columnName) { + this.columnName = columnName; + } + + @Override + public int hashCode() { + int hash = 7; + hash = 71 * hash + Objects.hashCode(this.schemaName); + hash = 71 * hash + Objects.hashCode(this.tableName); + hash = 71 * hash + Objects.hashCode(this.columnName); + return hash; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final ColumnHolder other = (ColumnHolder) obj; + if (!Objects.equals(this.schemaName, other.schemaName)) { + return false; + } + if (!Objects.equals(this.tableName, other.tableName)) { + return false; + } + return Objects.equals(this.columnName, other.columnName); + } + + @Override + public String toString() { + return String.format("%s.%s.%s", schemaName, tableName, columnName); + } +} diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/ConsistencyChecks.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/ConsistencyChecks.java index b76770dc3ddc98ace64d9e5b3c09496238bf1295..74aa20469a1db3371f566d8dfd10e87c2c502868 100644 --- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/ConsistencyChecks.java +++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/ConsistencyChecks.java @@ -22,24 +22,20 @@ */ package it.inaf.ia2.tsm; -import it.inaf.ia2.tsm.datalayer.DatabaseType; -import it.inaf.ia2.tsm.datalayer.DBWrapper; +import it.inaf.ia2.tsm.model.ColumnModel; import java.io.Serializable; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; -import javax.sql.DataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * - * TODO: Move queries into data layer classes. + * DataModel for consistency checking result. * * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} */ @@ -48,110 +44,35 @@ public class ConsistencyChecks implements Serializable { private static final long serialVersionUID = 4412404312756740093L; private final static Logger LOG = LoggerFactory.getLogger(ConsistencyChecks.class); - public static class UnexistingColumn implements Serializable { - - private static final long serialVersionUID = -4898369878807200093L; - - private final String completeTableName; - private final String columnName; - - private UnexistingColumn(String completeTableName, String columnName) { - this.completeTableName = completeTableName; - this.columnName = columnName; - } - - public String getCompleteTableName() { - return completeTableName; - } - - public String getColumnName() { - return columnName; - } - - @Override - public String toString() { - return String.format("%s.%s", completeTableName, columnName); - } - } - - public static class UnexistingKey implements Serializable { - - private static final long serialVersionUID = 7891439129072900628L; - - private final String keyId; - private final String fromTable; - private final String[] fromColumns; - private final String targetTable; - private final String[] targetColumns; - - private UnexistingKey(String keyId, String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) { - this.keyId = keyId; - this.fromTable = fromTable; - this.fromColumns = fromColumns; - this.targetTable = targetTable; - this.targetColumns = targetColumns; - } - - public String getKeyId() { - return keyId; - } - - public String getFromTable() { - return fromTable; - } - - public String[] getFromColumns() { - return fromColumns; - } - - public String getTargetTable() { - return targetTable; - } - - public String[] getTargetColumns() { - return targetColumns; - } - - private String getColumnsString(String[] columns) { - StringBuilder sb = new StringBuilder(); - boolean first = true; - for (String column : columns) { - if (!first) { - sb.append(","); - } - sb.append(column); - first = false; - } - return sb.toString(); - } - - @Override - public String toString() { - return String.format("[%s] %s(%s) -> %s(%s)", keyId, - fromTable, getColumnsString(fromColumns), - targetTable, getColumnsString(targetColumns)); - } - } - - private final List<InconsistentValue> inconsistencies; + private final List<InconsistentColumnProperty> inconsistencies; private final Set<String> unexisingSchemas; private final Set<String> unexisingTables; - private final List<UnexistingColumn> unexistingColumns; - private final List<UnexistingKey> unexistingKeys; + private final Set<ColumnHolder> unexistingColumns; + private final Set<KeyHolder> unexistingKeys; + private final Map<String, Set<String>> missingTables; + private final Map<String, Set<String>> tablesToAdd; + private final Map<ColumnHolder, ColumnModel> missingColumns; + private final Set<ColumnHolder> columnsToAdd; + private boolean missingObscore; + private boolean obscoreToAdd; public ConsistencyChecks() { inconsistencies = new ArrayList<>(); unexisingSchemas = new HashSet<>(); unexisingTables = new HashSet<>(); - unexistingColumns = new ArrayList<>(); - unexistingKeys = new ArrayList<>(); + unexistingColumns = new HashSet<>(); + unexistingKeys = new HashSet<>(); + missingTables = new HashMap<>(); + tablesToAdd = new HashMap<>(); + missingColumns = new HashMap<>(); + columnsToAdd = new HashSet<>(); } - public void addInconsistency(InconsistentValue problemDescription) { + public void addInconsistency(InconsistentColumnProperty problemDescription) { inconsistencies.add(problemDescription); } - public List<InconsistentValue> getInconsistencies() { + public List<InconsistentColumnProperty> getInconsistencies() { return inconsistencies; } @@ -171,192 +92,101 @@ public class ConsistencyChecks implements Serializable { unexisingTables.add(schemaName + "." + tableSimpleName); } - public List<UnexistingColumn> getUnexisingColumns() { + public Set<ColumnHolder> getUnexisingColumns() { return unexistingColumns; } - public void addUnexistingColumn(String completeTableName, String columnName) { - unexistingColumns.add(new UnexistingColumn(completeTableName, columnName)); + public void addUnexistingColumn(String schemaName, String tableName, String columnName) { + unexistingColumns.add(new ColumnHolder(schemaName, tableName, columnName)); } public void addUnexistingKey(String keyId, String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) { if (keyId == null) { throw new IllegalArgumentException("key_id can't be null"); } - unexistingKeys.add(new UnexistingKey(keyId, fromTable, fromColumns, targetTable, targetColumns)); + unexistingKeys.add(new KeyHolder(keyId, fromTable, fromColumns, targetTable, targetColumns)); } - public List<UnexistingKey> getUnexistingKeys() { + public Set<KeyHolder> getUnexistingKeys() { return unexistingKeys; } - private Set<String> getKeysToRemove(Connection conn, String tapSchemaNameEscaped, DatabaseType dbType, String like) throws SQLException { - Set<String> ret = new HashSet<>(); - String query = String.format("SELECT key_id from %s.%s WHERE from_table LIKE ? OR target_table LIKE ?", tapSchemaNameEscaped, TSMUtil.escapeName("keys", dbType)); - try (PreparedStatement ps = conn.prepareStatement(query)) { - ps.setString(1, like + "%"); - ps.setString(2, like + "%"); - LOG.debug("Executing query: {} [{}]", query, like); - try (ResultSet rs = ps.executeQuery()) { - while (rs.next()) { - ret.add(rs.getString("key_id")); - } - } - } - return ret; + public void addMissingColumn(ColumnHolder columnHolder, ColumnModel columnModel) { + // Removing table from unexisting columns set + unexistingColumns.remove(columnHolder); + missingColumns.put(columnHolder, columnModel); } - public boolean isInconsistent() { - return !inconsistencies.isEmpty() || !unexisingSchemas.isEmpty() || !unexisingTables.isEmpty() || !unexistingColumns.isEmpty(); + public Map<ColumnHolder, ColumnModel> getMissingColumns() { + return missingColumns; } - private void keysToRemoveFromUnexistingColumns(Connection conn, String tapSchemaNameEscaped, Set<String> keysToRemoveIds) throws SQLException { - for (UnexistingColumn unexistingColumn : unexistingColumns) { - - StringBuilder sb = new StringBuilder(); - sb.append("SELECT k.key_id AS key_id\n"); - sb.append("FROM "); - sb.append(tapSchemaNameEscaped); - sb.append(".`keys` k\n"); - sb.append("JOIN "); - sb.append(tapSchemaNameEscaped); - sb.append(".key_columns c ON k.key_id = c.key_id\n"); - sb.append("WHERE (k.from_table = ? AND c.from_column = ?) OR (k.target_table = ? AND c.target_column = ?)"); - - String query = sb.toString(); - - try (PreparedStatement ps = conn.prepareStatement(query)) { - ps.setString(1, unexistingColumn.getCompleteTableName()); - ps.setString(2, unexistingColumn.getColumnName()); - ps.setString(3, unexistingColumn.getCompleteTableName()); - ps.setString(4, unexistingColumn.getColumnName()); - - LOG.debug("Executing query {}", query); - - try (ResultSet rs = ps.executeQuery()) { - while (rs.next()) { - keysToRemoveIds.add(rs.getString("key_id")); - } - } - } + private void addTableToMap(String schemaName, String tableName, Map<String, Set<String>> map) { + Set<String> tables = map.get(schemaName); + if (tables == null) { + tables = new HashSet<>(); + map.put(schemaName, tables); } + tables.add(tableName); } - public void amendTapSchema(DBWrapper dbWrapper, TapSchema tapSchema) throws SQLException { - - Set<String> keysToRemoveIds = new HashSet<>(); + public void addMissingTable(String schemaName, String tableName) { + // Removing table from unexisting table set + unexisingTables.remove(String.format("%s.%s", schemaName, tableName)); - DatabaseType dbType = dbWrapper.getTapSchemaDatabaseType(); - DataSource dataSource = dbWrapper.getTapSchemaDataSource(); - - String tapSchemaNameEscaped = TSMUtil.escapeName(tapSchema.getName(), dbType); + // Removing table from unexisting columns set + Iterator<ColumnHolder> ite = unexistingColumns.iterator(); + while (ite.hasNext()) { + ColumnHolder ch = ite.next(); + if (ch.getSchemaName().equals(schemaName) && ch.getTableName().equals(tableName)) { + ite.remove(); + } + } - String query; - try (Connection conn = dataSource.getConnection()) { + addTableToMap(schemaName, tableName, missingTables); + } - for (String schema : unexisingSchemas) { - keysToRemoveIds.addAll(getKeysToRemove(conn, tapSchemaNameEscaped, dbType, schema)); - } - for (String table : unexisingTables) { - keysToRemoveIds.addAll(getKeysToRemove(conn, tapSchemaNameEscaped, dbType, table)); - } + public void addTableToAdd(String schemaName, String tableName) { + addTableToMap(schemaName, tableName, tablesToAdd); + } - keysToRemoveFromUnexistingColumns(conn, tapSchemaNameEscaped, keysToRemoveIds); + public Map<String, Set<String>> getTablesToAdd() { + return tablesToAdd; + } - for (UnexistingKey unexistingKey : unexistingKeys) { - keysToRemoveIds.add(unexistingKey.getKeyId()); - } + public Map<String, Set<String>> getMissingTables() { + return missingTables; + } - conn.setAutoCommit(false); - LOG.debug("Starting transaction"); + public void addColumnToAdd(ColumnHolder columnHolder) { + columnsToAdd.add(columnHolder); + } - try { - // Removing all key_columns - for (String keyId : keysToRemoveIds) { - query = String.format("DELETE FROM %s.%s WHERE key_id = ?", tapSchemaNameEscaped, TSMUtil.escapeName("key_columns", dbType)); - try (PreparedStatement ps = conn.prepareStatement(query)) { - ps.setString(1, keyId); - LOG.debug("Executing query {} [{}]", query, keyId); - ps.executeUpdate(); - } - } + public Set<ColumnHolder> getColumnsToAdd() { + return columnsToAdd; + } - // Removing all keys - for (String keyId : keysToRemoveIds) { - query = String.format("DELETE FROM %s.%s WHERE key_id = ?", tapSchemaNameEscaped, TSMUtil.escapeName("keys", dbType)); - try (PreparedStatement ps = conn.prepareStatement(query)) { - ps.setString(1, keyId); - LOG.debug("Executing query {} [{}]", query, keyId); - ps.executeUpdate(); - } - } + public boolean isMissingObscore() { + return missingObscore; + } - // Removing all columns - for (UnexistingColumn unexistingColumn : unexistingColumns) { - query = String.format("DELETE FROM %s.%s WHERE table_name = ? AND column_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType)); - try (PreparedStatement ps = conn.prepareStatement(query)) { - ps.setString(1, unexistingColumn.getCompleteTableName()); - ps.setString(2, unexistingColumn.getColumnName()); - LOG.debug("Executing query {} [{}, {}]", query, unexistingColumn.getCompleteTableName(), unexistingColumn.getColumnName()); - ps.executeUpdate(); - } - } - for (String table : unexisingTables) { - query = String.format("DELETE FROM %s.%s WHERE table_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType)); - try (PreparedStatement ps = conn.prepareStatement(query)) { - ps.setString(1, table); - LOG.debug("Executing query {} [{}]", query, table); - ps.executeUpdate(); - } - } - for (String schema : unexisingSchemas) { - query = String.format("DELETE FROM %s.%s WHERE table_name LIKE ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType)); - try (PreparedStatement ps = conn.prepareStatement(query)) { - ps.setString(1, schema + "%"); - LOG.debug("Executing query {} [{}%]", query, schema); - ps.executeUpdate(); - } - } + public void setMissingObscore(boolean missingObscore) { + this.missingObscore = missingObscore; + } - // Removing all tables - for (String table : unexisingTables) { - query = String.format("DELETE FROM %s.%s WHERE table_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("tables", dbType)); - try (PreparedStatement ps = conn.prepareStatement(query)) { - ps.setString(1, table); - LOG.debug("Executing query {} [{}]", query, table); - ps.executeUpdate(); - } - } - for (String schema : unexisingSchemas) { - query = String.format("DELETE FROM %s.%s WHERE schema_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("tables", dbType)); - try (PreparedStatement ps = conn.prepareStatement(query)) { - ps.setString(1, schema); - LOG.debug("Executing query {} [{}]", query, schema); - ps.executeUpdate(); - } - } + public boolean isObscoreToAdd() { + return obscoreToAdd; + } - // Removing all schemas - for (String schema : unexisingSchemas) { - query = String.format("DELETE FROM %s.%s WHERE schema_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("schemas", dbType)); - try (PreparedStatement ps = conn.prepareStatement(query)) { - ps.setString(1, schema); - LOG.debug("Executing query {} [{}]", query, schema); - ps.executeUpdate(); - } - } + public void setObscoreToAdd(boolean obscoreToAdd) { + this.obscoreToAdd = obscoreToAdd; + } - conn.commit(); - } catch (SQLException e) { - LOG.error("Exception detected. Executing rollback!", e); - try { - conn.rollback(); - conn.setAutoCommit(true); - } catch (SQLException er) { - LOG.error("Exception during rollback", er); - throw er; - } - } - } + public boolean isInconsistent() { + return !inconsistencies.isEmpty() + || !unexisingSchemas.isEmpty() || !unexisingTables.isEmpty() || !unexistingColumns.isEmpty() + || !missingTables.isEmpty() || !missingColumns.isEmpty() + || !columnsToAdd.isEmpty() || !tablesToAdd.isEmpty() + || obscoreToAdd || missingObscore; } } diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/EntitiesContainer.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/EntitiesContainer.java index 8c10dd114d4c03ed822234ae597203bdb4b6dba2..01c9c95b36a1da65efc03acf9421da7e86fba646 100644 --- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/EntitiesContainer.java +++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/EntitiesContainer.java @@ -64,6 +64,8 @@ public interface EntitiesContainer<T extends ChildEntity> { */ List<String> getAddableChildrenNames(); + boolean isAddable(String childName); + /** * Retrieve a list of children filtering them by a set of possible * status.<br> diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentValue.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentColumnProperty.java similarity index 62% rename from TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentValue.java rename to TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentColumnProperty.java index d7e6c11597feaeedceb7d8868f6c051ec3040f7e..e368beda67cdd8ffa3e52b74006b66ee7813bd87 100644 --- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentValue.java +++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentColumnProperty.java @@ -22,40 +22,43 @@ */ package it.inaf.ia2.tsm; +import java.io.Serializable; + /** * * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} */ -public class InconsistentValue { - - private final String tapSchemaEntityType; +public class InconsistentColumnProperty implements Serializable { - private final String tapSchemaEntityDescription; + private static final long serialVersionUID = -5145865322582594970L; - private final String wrongPropertyName; + private String tableCompleteName; + private String columnName; + private String key; + private Object currentValue; + private Object correctValue; - private final Object currentValue; - - private final Object correctValue; + private InconsistentColumnProperty() { + } - public InconsistentValue(String tapSchemaEntityType, String tapSchemaEntityDescription, String wrongPropertyName, Object currentValue, Object correctValue) { - this.tapSchemaEntityType = tapSchemaEntityType; - this.tapSchemaEntityDescription = tapSchemaEntityDescription; - this.wrongPropertyName = wrongPropertyName; + public InconsistentColumnProperty(String tableCompleteName, String columnName, String key, Object currentValue, Object correctValue) { + this.tableCompleteName = tableCompleteName; + this.columnName = columnName; + this.key = key; this.currentValue = currentValue; this.correctValue = correctValue; } - public String getTapSchemaEntityType() { - return tapSchemaEntityType; + public String getTableCompleteName() { + return tableCompleteName; } - public String getTapSchemaEntityDescription() { - return tapSchemaEntityDescription; + public String getColumnName() { + return columnName; } - public String getWrongPropertyName() { - return wrongPropertyName; + public String getKey() { + return key; } public Object getCurrentValue() { diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/KeyHolder.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/KeyHolder.java new file mode 100644 index 0000000000000000000000000000000000000000..a8fff372fe2cefc67fb874ce5019368e7ae2c47e --- /dev/null +++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/KeyHolder.java @@ -0,0 +1,91 @@ +/* + * _____________________________________________________________________________ + * + * INAF - OATS National Institute for Astrophysics - Astronomical Observatory of + * Trieste INAF - IA2 Italian Center for Astronomical Archives + * _____________________________________________________________________________ + * + * Copyright (C) 2017 Istituto Nazionale di Astrofisica + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License Version 3 as published by the + * Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package it.inaf.ia2.tsm; + +import java.io.Serializable; + +/** + * + * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} + */ +public class KeyHolder implements Serializable { + + private static final long serialVersionUID = 3711149068153684261L; + + private String keyId; + private String fromTable; + private String[] fromColumns; + private String targetTable; + private String[] targetColumns; + + private KeyHolder() { + } + + public KeyHolder(String keyId, String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) { + this.keyId = keyId; + this.fromTable = fromTable; + this.fromColumns = fromColumns; + this.targetTable = targetTable; + this.targetColumns = targetColumns; + } + + public String getKeyId() { + return keyId; + } + + public String getFromTable() { + return fromTable; + } + + public String[] getFromColumns() { + return fromColumns; + } + + public String getTargetTable() { + return targetTable; + } + + public String[] getTargetColumns() { + return targetColumns; + } + + private String getColumnsString(String[] columns) { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (String column : columns) { + if (!first) { + sb.append(","); + } + sb.append(column); + first = false; + } + return sb.toString(); + } + + @Override + public String toString() { + return String.format("[%s] %s(%s) -> %s(%s)", keyId, + fromTable, getColumnsString(fromColumns), + targetTable, getColumnsString(targetColumns)); + } +} diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/Schema.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/Schema.java index ad3df64209aed2c345a9dcb8766557b935577cd2..3a5e9374ed391f0b35aadb068152b67070d6935a 100644 --- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/Schema.java +++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/Schema.java @@ -175,6 +175,11 @@ public class Schema extends ChildEntity<TapSchema> implements EntitiesContainer< return TSMUtil.getAddableChildrenNames(tables); } + @Override + public boolean isAddable(String childName) { + return tables.containsKey(childName); + } + /** * {@inheritDoc} */ diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TSMUtil.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TSMUtil.java index fef324ec5aec28067d9e508f393bd54faedc3c95..451f3bd2322bf5cf6eb39b753795258c0f33999f 100644 --- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TSMUtil.java +++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TSMUtil.java @@ -203,7 +203,7 @@ public class TSMUtil { return null; } - public static String getNaturalLangueName(TapSchemaEntity entity) { + public static String getNaturalLanguageName(TapSchemaEntity entity) { if (entity instanceof Schema) { return "schema"; } else if (entity instanceof Table) { diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/Table.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/Table.java index 0827988457fbfed0b5478b6202978e79c16b9753..a5a4a83427dc9abcce15bad8b1362f0414e198a8 100644 --- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/Table.java +++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/Table.java @@ -192,6 +192,11 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu return TSMUtil.getAddableChildrenNames(columns); } + @Override + public boolean isAddable(String childName) { + return columns.containsKey(childName); + } + @Override public List<Column> getAddedChildren() { return getChildren(Status.ADDED_PERSISTED, Status.ADDED_NOT_PERSISTED); diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchema.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchema.java index e16e9de962bf9b2254ee68e3b2533f156e1a7a0c..bac34d5e496dd2f42812d5da31717276cc237731 100644 --- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchema.java +++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchema.java @@ -37,10 +37,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.TreeMap; -import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,7 +62,6 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable { private final Map<String, Schema> schemas; private final Set<Key> allKeys; - private final ConsistencyChecks consistencyChecks; private boolean loading; private DBWrapper dbWrapper; @@ -75,6 +72,8 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable { private transient DBBroker sourceDBBroker; private transient DBBroker tapSchemaDBBroker; + private ConsistencyChecks consistencyChecks; + public final DBBroker getSourceDBBroker() { if (sourceDBBroker == null) { sourceDBBroker = DBBrokerFactory.getDBBroker(dbWrapper.getSourceDataSourceWrapper(), dataTypeMode); @@ -93,190 +92,38 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable { // for serialization schemas = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); allKeys = new HashSet<>(); - consistencyChecks = new ConsistencyChecks(); + //consistencyChecks = new ConsistencyChecks(); } public TapSchema(DBWrapper dbWrapper, TapSchemaSettings settings, boolean exists) throws SQLException { this(); - loading = true; this.dbWrapper = dbWrapper; this.exists = exists; this.settings = settings; dataTypeMode = getTapSchemaModel().getDataTypeMode(); + load(); + } + + public final void load() throws SQLException { + + loading = true; // Initializing schemas map + schemas.clear(); for (String schemaName : getSourceDBBroker().getAllSchemaNames()) { schemas.put(schemaName, null); } schemas.put(settings.getTapSchemaName(), null); // the TAP_SCHEMA contains itself if (exists) { - loadSavedTapSchema(); + consistencyChecks = TapSchemaLoader.loadExistingTapSchema((this)); } loading = false; - checkKeys(); - } - - private void loadSavedProperties(TapSchemaEntity tapSchemaEntity, Map<String, Object> savedProperties) { - for (Map.Entry<String, Object> entry : savedProperties.entrySet()) { - String key = entry.getKey(); - Object savedValue = entry.getValue(); - Object currentValue = entry.getValue(); - EntityProperty ep = tapSchemaEntity.getProperty(key); - - if (!ep.isUpdatable() && !Objects.equals(savedValue, currentValue)) { - - InconsistentValue inconsistentValue = new InconsistentValue( - TSMUtil.getNaturalLangueName(tapSchemaEntity), - TSMUtil.getName(tapSchemaEntity), - key, - savedValue, - savedValue - ); - - consistencyChecks.addInconsistency(inconsistentValue); - } - - tapSchemaEntity.initProperty(key, savedValue); - } - } - - /** - * Loads saved TAP_SCHEMA data and performs consistency checking. - * - * @throws SQLException - */ - private void loadSavedTapSchema() throws SQLException { - DBBroker broker = getTapSchemaDBBroker(); - - // Schemata - for (Map<String, Object> schemaProps : broker.getSavedItems(getName(), getTableModel(SCHEMAS_TABLE))) { - String schemaName = (String) schemaProps.get(Schema.SCHEMA_NAME_KEY); - Schema schema = addChild(schemaName); - if (schema == null) { - consistencyChecks.addUnexistingSchema(schemaName); - } else { - loadSavedProperties(schema, schemaProps); - schema.setStatus(Status.ADDED_PERSISTED); - } - } - - // Tables - for (Map<String, Object> tableProps : broker.getSavedItems(getName(), getTableModel(TABLES_TABLE))) { - String tableCompleteName = (String) tableProps.get(Table.TABLE_NAME_KEY); - String[] tableNameSplit = tableCompleteName.split(Pattern.quote(".")); - String schemaName = tableNameSplit[0]; - String tableName = tableNameSplit[1]; - Schema schema = getChild(schemaName, Status.ADDED_PERSISTED); - if (schema == null) { - consistencyChecks.addUnexistingSchema(schemaName); - } else { - Table table = schema.addChild(tableName); - if (table == null) { - consistencyChecks.addUnexistingTable(schemaName, tableName); - } else { - loadSavedProperties(table, tableProps); - table.setStatus(Status.ADDED_PERSISTED); - } - } - } - - // Columns - for (Map<String, Object> columnProps : broker.getSavedItems(getName(), getTableModel(COLUMNS_TABLE))) { - String tableCompleteName = (String) columnProps.get(Column.TABLE_NAME_KEY); - String[] tableNameSplit = tableCompleteName.split(Pattern.quote(".")); - String schemaName = tableNameSplit[0]; - String tableName = tableNameSplit[1]; - String columnName = (String) columnProps.get(Column.COLUMN_NAME_KEY); - Schema schema = getChild(schemaName, Status.ADDED_PERSISTED); - if (schema == null) { - consistencyChecks.addUnexistingSchema(schemaName); - } else { - Table table = schema.getChild(tableName, Status.ADDED_PERSISTED); - if (table == null) { - consistencyChecks.addUnexistingTable(schemaName, tableName); - } else { - Column column = table.addChild(columnName); - if (column == null) { - consistencyChecks.addUnexistingColumn(tableCompleteName, columnName); - } else { - loadSavedProperties(column, columnProps); - column.setStatus(Status.ADDED_PERSISTED); - } - } - } - } - // Keys - List<Map<String, Object>> keysProps = broker.getSavedItems(getName(), getTableModel(KEYS_TABLE)); - List<Map<String, Object>> keysColumnsProps = broker.getSavedItems(getName(), getTableModel(KEY_COLUMNS_TABLE)); - for (Map<String, Object> keyProp : keysProps) { - - String fromTable = (String) keyProp.get(Key.FROM_TABLE_KEY); - String targetTable = (String) keyProp.get(Key.TARGET_TABLE_KEY); - String keyId = (String) keyProp.get(Key.ID_KEY); - assert keyId != null; - - List<Map<String, Object>> kcPropsById = new ArrayList<>(); - for (Map<String, Object> kcp : keysColumnsProps) { - String keyColumnId = (String) kcp.get(KeyColumn.KEY_ID_KEY); - assert keyColumnId != null; - if (keyColumnId.equals(keyId)) { - kcPropsById.add(kcp); - } - } - - // Searching the key - boolean keyFound = false; - for (Key key : allKeys) { - if (key.getFromTableCompleteName().equals(fromTable) && key.getTargetTableCompleteName().equals(targetTable)) { - // Search the key columns having proper key id - - // Verifying the matching - List<KeyColumn> matchedKeyColumns = new ArrayList<>(); - if (kcPropsById.size() == key.getKeyColumns().size()) { - for (Map<String, Object> kcp : kcPropsById) { - String fromColumn = (String) kcp.get(KeyColumn.FROM_COLUMN_KEY); - String targetColumn = (String) kcp.get(KeyColumn.TARGET_COLUMN_KEY); - for (KeyColumn keyColumn : key.getKeyColumns()) { - if (keyColumn.getFromColumn().equals(fromColumn) - && keyColumn.getTargetColumn().equals(targetColumn)) { - matchedKeyColumns.add(keyColumn); - } - } - } - } - if (kcPropsById.size() == matchedKeyColumns.size()) { - keyFound = true; - int index = 0; - loadSavedProperties(key, keyProp); - for (Map<String, Object> kcp : kcPropsById) { - KeyColumn kc = matchedKeyColumns.get(index); - loadSavedProperties(kc, kcp); - index++; - } - } - } - } - - if (!keyFound) { - boolean setKeyToRemove = true; - if (setKeyToRemove) { - String[] fromColumns = new String[kcPropsById.size()]; - String[] targetColumns = new String[kcPropsById.size()]; - int i = 0; - for (Map<String, Object> kcp : kcPropsById) { - fromColumns[i] = (String) kcp.get(KeyColumn.FROM_COLUMN_KEY); - targetColumns[i] = (String) kcp.get(KeyColumn.TARGET_COLUMN_KEY); - i++; - } - consistencyChecks.addUnexistingKey(keyId, fromTable, fromColumns, targetTable, targetColumns); - } - } - } + checkKeys(); } public DBBroker getDBBroker(String schemaName) { @@ -401,6 +248,11 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable { return TSMUtil.getAddableChildrenNames(schemas); } + @Override + public boolean isAddable(String childName) { + return schemas.containsKey(childName); + } + /** * {@inheritDoc} */ @@ -436,7 +288,7 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable { return null; } - private void addEntireSchema(String schemaName) throws SQLException { + public void addEntireSchema(String schemaName) throws SQLException { Schema schema = addChild(schemaName); for (String tableName : schema.getAddableChildrenNames()) { Table table = schema.addChild(tableName); @@ -462,16 +314,7 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable { fillColumnProperties(tapSchemaModel, getName()); if (settings.isHasObscore()) { - SchemaModel ivoaSchemaModel = getIvoaSchemaModel(); - - // ivoa schema has to be created into source database - getSourceDBBroker().createIvoaSchemaStructure(ivoaSchemaModel); - // Initializing ivoa schema slot in schemata maps - schemas.put(ivoaSchemaModel.getName(), null); - - // Add ivoa schema into TAP_SCHEMA - addEntireSchema(ivoaSchemaModel.getName()); - fillColumnDescriptionsAndStd(ivoaSchemaModel); + createAndAddIvoaSchema(); } } @@ -480,7 +323,21 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable { broker.save(this); exists = true; - consistencyChecks.getInconsistencies().clear(); + // Clean inconsistency + consistencyChecks = null; + } + + public void createAndAddIvoaSchema() throws SQLException { + SchemaModel ivoaSchemaModel = getIvoaSchemaModel(); + + // ivoa schema has to be created into source database + getSourceDBBroker().createIvoaSchemaStructure(ivoaSchemaModel); + // Initializing ivoa schema slot in schemata maps + schemas.put(ivoaSchemaModel.getName(), null); + + // Add ivoa schema into TAP_SCHEMA + addEntireSchema(ivoaSchemaModel.getName()); + fillColumnsProperties(ivoaSchemaModel); } /** @@ -725,7 +582,7 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable { } } - private void fillColumnDescriptionsAndStd(SchemaModel schemaModel) { + public void fillColumnsProperties(SchemaModel schemaModel) { fillColumnProperties(schemaModel, schemaModel.getName()); } } diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchemaLoader.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchemaLoader.java new file mode 100644 index 0000000000000000000000000000000000000000..beebaec81792a316deaacdc414deb3b5fba266d2 --- /dev/null +++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchemaLoader.java @@ -0,0 +1,350 @@ +/* + * _____________________________________________________________________________ + * + * INAF - OATS National Institute for Astrophysics - Astronomical Observatory of + * Trieste INAF - IA2 Italian Center for Astronomical Archives + * _____________________________________________________________________________ + * + * Copyright (C) 2017 Istituto Nazionale di Astrofisica + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License Version 3 as published by the + * Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package it.inaf.ia2.tsm; + +import static it.inaf.ia2.tsm.TapSchema.COLUMNS_TABLE; +import static it.inaf.ia2.tsm.TapSchema.KEYS_TABLE; +import static it.inaf.ia2.tsm.TapSchema.KEY_COLUMNS_TABLE; +import static it.inaf.ia2.tsm.TapSchema.SCHEMAS_TABLE; +import static it.inaf.ia2.tsm.TapSchema.TABLES_TABLE; +import it.inaf.ia2.tsm.datalayer.DBBroker; +import it.inaf.ia2.tsm.datalayer.DataTypeMode; +import it.inaf.ia2.tsm.model.ColumnModel; +import it.inaf.ia2.tsm.model.SchemaModel; +import it.inaf.ia2.tsm.model.TableModel; +import it.inaf.ia2.tsm.model.TypesMapping; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Pattern; + +/** + * Loads the TAP_SCHEMA and performs consistency checking. + * + * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} + */ +public class TapSchemaLoader { + + private final TapSchema tapSchema; + private final DBBroker tapSchemaDBBroker; + private ConsistencyChecks consistencyChecks; + + public static ConsistencyChecks loadExistingTapSchema(TapSchema tapSchema) throws SQLException { + TapSchemaLoader loader = new TapSchemaLoader(tapSchema); + return loader.loadExistingTapSchema(); + } + + private TapSchemaLoader(TapSchema tapSchema) { + this.tapSchema = tapSchema; + tapSchemaDBBroker = tapSchema.getTapSchemaDBBroker(); + } + + private ConsistencyChecks loadExistingTapSchema() throws SQLException { + + consistencyChecks = new ConsistencyChecks(); + + loadAndCheckSchemata(); + loadAndCheckTables(); + loadAndCheckColumns(); + loadAndCheckKeys(); + + checkModel(tapSchema.getTapSchemaModel()); + checkObscore(); + + return consistencyChecks; + } + + private void checkObscore() { + if (tapSchema.isHasObscore()) { + String ivoaSchemaName = tapSchema.getIvoaSchemaModel().getName(); + if (tapSchema.getChild(ivoaSchemaName) == null) { + if (tapSchema.isAddable(ivoaSchemaName)) { + consistencyChecks.setObscoreToAdd(true); + } else { + consistencyChecks.setMissingObscore(true); + } + } else { + checkModel(tapSchema.getIvoaSchemaModel()); + } + } + } + + private void checkModel(SchemaModel schemaModel) { + + String schemaName = schemaModel.getName(); + if ("tap_schema".equals(schemaName)) { + schemaName = tapSchema.getName(); + } + + Schema schema = tapSchema.getChild(schemaName); + + for (TableModel tableModel : schemaModel.getTables()) { + + String tableName = tableModel.getName(); + Table table = schema.getChild(tableName); + + if (table == null) { + if (schema.isAddable(tableModel.getName())) { + consistencyChecks.addTableToAdd(schema.getName(), tableName); + } else { + consistencyChecks.addMissingTable(schema.getName(), tableName); + } + } else { + + for (ColumnModel columnModel : tableModel.getColumns()) { + + Column column = table.getChild(columnModel.getName()); + + if (columnModel.isMandatory() && column == null) { + + ColumnHolder ch = new ColumnHolder(schemaName, + table.getName(), columnModel.getName()); + + if (table.isAddable(columnModel.getName())) { + consistencyChecks.addColumnToAdd(ch); + } else { + consistencyChecks.addMissingColumn(ch, columnModel); + } + } + } + } + } + } + + private boolean equalsOneOf(String key, String... names) { + for (String name : names) { + if (name.equals(key)) { + return true; + } + } + return false; + } + + private boolean hasFixedValue(Column column, String key) { + EntityProperty ep = column.getProperty(key); + if (!ep.isUpdatable()) { + return true; + } + Schema parentSchema = column.getParent().getParent(); + if (parentSchema.getName().equals(tapSchema.getName())) { + return equalsOneOf(key, Column.STD_KEY, Column.PRINCIPAL_KEY); + } else if (parentSchema.getName().equals(tapSchema.getIvoaSchemaModel().getName())) { + return equalsOneOf(key, Column.STD_KEY, Column.PRINCIPAL_KEY, + Column.UCD_KEY, Column.UNIT_KEY, Column.UTYPE_KEY, Column.DATATYPE_KEY); + } + return false; + } + + private Object getCorrectValue(Column column, String key) { + + EntityProperty ep = column.getProperty(key); + if (!ep.isUpdatable()) { + return column.getMetadata(key); + } + + Schema parentSchema = column.getParent().getParent(); + SchemaModel schemaModel; + if (parentSchema.getName().equals(tapSchema.getName())) { + schemaModel = tapSchema.getTapSchemaModel(); + } else { + schemaModel = tapSchema.getIvoaSchemaModel(); + } + + ColumnModel columnModel = schemaModel.getTable(column.getParent().getName()).get(column.getName()); + switch (key) { + case Column.STD_KEY: + return columnModel.isStandard(); + case Column.PRINCIPAL_KEY: + return columnModel.isPrincipal(); + case Column.UCD_KEY: + return columnModel.getUcd(); + case Column.UNIT_KEY: + return columnModel.getUnit(); + case Column.UTYPE_KEY: + return columnModel.getUtype(); + case Column.DATATYPE_KEY: + return TypesMapping.getDataType(columnModel.getType(), tapSchema.getDataTypeMode()); + } + + throw new RuntimeException("Unable to retrieve correct value for " + key); + } + + private void loadSavedProperties(TapSchemaEntity tapSchemaEntity, Map<String, Object> savedProperties) { + for (Map.Entry<String, Object> entry : savedProperties.entrySet()) { + + String key = entry.getKey(); + Object savedValue = entry.getValue(); + + if (tapSchemaEntity instanceof Column) { + Column column = (Column) tapSchemaEntity; + if (hasFixedValue(column, key)) { + Object correctValue = getCorrectValue(column, key); + if (!Objects.equals(savedValue, correctValue)) { + InconsistentColumnProperty inconsistentValue = new InconsistentColumnProperty( + column.getTableCompleteName(), + column.getName(), + key, + savedValue, + correctValue + ); + consistencyChecks.addInconsistency(inconsistentValue); + } + } + } + + tapSchemaEntity.initProperty(key, savedValue); + } + } + + private void loadAndCheckSchemata() throws SQLException { + for (Map<String, Object> schemaProps : tapSchemaDBBroker.getSavedItems(tapSchema.getName(), tapSchema.getTableModel(SCHEMAS_TABLE))) { + String schemaName = (String) schemaProps.get(Schema.SCHEMA_NAME_KEY); + Schema schema = tapSchema.addChild(schemaName); + if (schema == null) { + consistencyChecks.addUnexistingSchema(schemaName); + } else { + loadSavedProperties(schema, schemaProps); + schema.setStatus(Status.ADDED_PERSISTED); + } + } + } + + private void loadAndCheckTables() throws SQLException { + for (Map<String, Object> tableProps : tapSchemaDBBroker.getSavedItems(tapSchema.getName(), tapSchema.getTableModel(TABLES_TABLE))) { + String tableCompleteName = (String) tableProps.get(Table.TABLE_NAME_KEY); + String[] tableNameSplit = tableCompleteName.split(Pattern.quote(".")); + String schemaName = tableNameSplit[0]; + String tableName = tableNameSplit[1]; + Schema schema = tapSchema.getChild(schemaName, Status.ADDED_PERSISTED); + if (schema == null) { + consistencyChecks.addUnexistingSchema(schemaName); + } else { + Table table = schema.addChild(tableName); + if (table == null) { + consistencyChecks.addUnexistingTable(schemaName, tableName); + } else { + loadSavedProperties(table, tableProps); + table.setStatus(Status.ADDED_PERSISTED); + } + } + } + } + + private void loadAndCheckColumns() throws SQLException { + for (Map<String, Object> columnProps : tapSchemaDBBroker.getSavedItems(tapSchema.getName(), tapSchema.getTableModel(COLUMNS_TABLE))) { + String tableCompleteName = (String) columnProps.get(Column.TABLE_NAME_KEY); + String[] tableNameSplit = tableCompleteName.split(Pattern.quote(".")); + String schemaName = tableNameSplit[0]; + String tableName = tableNameSplit[1]; + String columnName = (String) columnProps.get(Column.COLUMN_NAME_KEY); + Schema schema = tapSchema.getChild(schemaName, Status.ADDED_PERSISTED); + if (schema == null) { + consistencyChecks.addUnexistingSchema(schemaName); + } else { + Table table = schema.getChild(tableName, Status.ADDED_PERSISTED); + if (table == null) { + consistencyChecks.addUnexistingTable(schemaName, tableName); + } else { + Column column = table.addChild(columnName); + if (column == null) { + consistencyChecks.addUnexistingColumn(schemaName, tableName, columnName); + } else { + loadSavedProperties(column, columnProps); + column.setStatus(Status.ADDED_PERSISTED); + } + } + } + } + } + + private void loadAndCheckKeys() throws SQLException { + List<Map<String, Object>> keysProps = tapSchemaDBBroker.getSavedItems(tapSchema.getName(), tapSchema.getTableModel(KEYS_TABLE)); + List<Map<String, Object>> keysColumnsProps = tapSchemaDBBroker.getSavedItems(tapSchema.getName(), tapSchema.getTableModel(KEY_COLUMNS_TABLE)); + for (Map<String, Object> keyProp : keysProps) { + + String fromTable = (String) keyProp.get(Key.FROM_TABLE_KEY); + String targetTable = (String) keyProp.get(Key.TARGET_TABLE_KEY); + String keyId = (String) keyProp.get(Key.ID_KEY); + assert keyId != null; + + List<Map<String, Object>> kcPropsById = new ArrayList<>(); + for (Map<String, Object> kcp : keysColumnsProps) { + String keyColumnId = (String) kcp.get(KeyColumn.KEY_ID_KEY); + assert keyColumnId != null; + if (keyColumnId.equals(keyId)) { + kcPropsById.add(kcp); + } + } + + // Searching the key + boolean keyFound = false; + for (Key key : tapSchema.getAllKeys()) { + if (key.getFromTableCompleteName().equals(fromTable) && key.getTargetTableCompleteName().equals(targetTable)) { + // Search the key columns having proper key id + + // Verifying the matching + List<KeyColumn> matchedKeyColumns = new ArrayList<>(); + if (kcPropsById.size() == key.getKeyColumns().size()) { + for (Map<String, Object> kcp : kcPropsById) { + String fromColumn = (String) kcp.get(KeyColumn.FROM_COLUMN_KEY); + String targetColumn = (String) kcp.get(KeyColumn.TARGET_COLUMN_KEY); + for (KeyColumn keyColumn : key.getKeyColumns()) { + if (keyColumn.getFromColumn().equals(fromColumn) + && keyColumn.getTargetColumn().equals(targetColumn)) { + matchedKeyColumns.add(keyColumn); + } + } + } + } + if (kcPropsById.size() == matchedKeyColumns.size()) { + keyFound = true; + int index = 0; + loadSavedProperties(key, keyProp); + for (Map<String, Object> kcp : kcPropsById) { + KeyColumn kc = matchedKeyColumns.get(index); + loadSavedProperties(kc, kcp); + index++; + } + } + } + } + + if (!keyFound) { + boolean setKeyToRemove = true; + if (setKeyToRemove) { + String[] fromColumns = new String[kcPropsById.size()]; + String[] targetColumns = new String[kcPropsById.size()]; + int i = 0; + for (Map<String, Object> kcp : kcPropsById) { + fromColumns[i] = (String) kcp.get(KeyColumn.FROM_COLUMN_KEY); + targetColumns[i] = (String) kcp.get(KeyColumn.TARGET_COLUMN_KEY); + i++; + } + consistencyChecks.addUnexistingKey(keyId, fromTable, fromColumns, targetTable, targetColumns); + } + } + } + } +} diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchemaMender.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchemaMender.java new file mode 100644 index 0000000000000000000000000000000000000000..18f1152e4aafda422bb30ddb9c58dcab71fda3d2 --- /dev/null +++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchemaMender.java @@ -0,0 +1,157 @@ +/* + * _____________________________________________________________________________ + * + * INAF - OATS National Institute for Astrophysics - Astronomical Observatory of + * Trieste INAF - IA2 Italian Center for Astronomical Archives + * _____________________________________________________________________________ + * + * Copyright (C) 2017 Istituto Nazionale di Astrofisica + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License Version 3 as published by the + * Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package it.inaf.ia2.tsm; + +import it.inaf.ia2.tsm.datalayer.DBBroker; +import it.inaf.ia2.tsm.model.ColumnModel; +import it.inaf.ia2.tsm.model.SchemaModel; +import it.inaf.ia2.tsm.model.TableModel; +import java.sql.SQLException; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * This fixes TAP_SCHEMA inconsistencies. + * + * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} + */ +public class TapSchemaMender { + + private final TapSchema tapSchema; + private final ConsistencyChecks consistencyChecks; + + public TapSchemaMender(TapSchema tapSchema) { + this.tapSchema = tapSchema; + consistencyChecks = tapSchema.getConsistencyChecks(); + } + + public static void amendTapSchema(TapSchema tapSchema) throws SQLException { + int counter = 0; + while (tapSchema.getConsistencyChecks() != null && tapSchema.getConsistencyChecks().isInconsistent()) { + new TapSchemaMender(tapSchema).amendTapSchema(); + tapSchema.load(); + counter++; + if (counter > 20) { + throw new RuntimeException("Unable to amend TAP_SCHEMA"); + } + } + } + + private void amendTapSchema() throws SQLException { + + fixObscore(); + createMissingTables(); + createMissingColumns(); + deleteUnexistingEntities(); + addUnaddedTables(); + addUnaddedColumns(); + fixInconsistentValues(); + } + + private void fixObscore() throws SQLException { + + SchemaModel ivoaSchemaModel = tapSchema.getIvoaSchemaModel(); + + if (tapSchema.isHasObscore()) { + if (consistencyChecks.isMissingObscore()) { + tapSchema.createAndAddIvoaSchema(); + } + if (consistencyChecks.isMissingObscore() + || consistencyChecks.isObscoreToAdd()) { + tapSchema.addEntireSchema(ivoaSchemaModel.getName()); + tapSchema.fillColumnsProperties(ivoaSchemaModel); + } + } + } + + private void createMissingTables() throws SQLException { + for (Map.Entry<String, Set<String>> entry : consistencyChecks.getMissingTables().entrySet()) { + String schemaName = entry.getKey(); + DBBroker broker = tapSchema.getDBBroker(schemaName); + for (String tableName : entry.getValue()) { + SchemaModel schemaModel = schemaName.equals(tapSchema.getName()) + ? tapSchema.getTapSchemaModel() : tapSchema.getIvoaSchemaModel(); + TableModel tableModel = schemaModel.getTable(tableName); + broker.createTable(schemaName, tableModel); + } + } + } + + private void createMissingColumns() throws SQLException { + for (Map.Entry<ColumnHolder, ColumnModel> entry : consistencyChecks.getMissingColumns().entrySet()) { + ColumnHolder columnHolder = entry.getKey(); + ColumnModel columnModel = entry.getValue(); + DBBroker broker = tapSchema.getDBBroker(columnHolder.getSchemaName()); + broker.addColumn(columnHolder, columnModel); + } + } + + private void deleteUnexistingEntities() throws SQLException { + String tapSchemaName = tapSchema.getName(); + DBBroker tapSchemaDBBroker = tapSchema.getTapSchemaDBBroker(); + + Set<String> keysToRemoveIds = new HashSet<>(); + + for (String schema : consistencyChecks.getUnexisingSchemas()) { + keysToRemoveIds.addAll(tapSchemaDBBroker.getKeysToRemove(tapSchemaName, schema)); + } + for (String table : consistencyChecks.getUnexisingTables()) { + keysToRemoveIds.addAll(tapSchemaDBBroker.getKeysToRemove(tapSchemaName, table)); + } + for (ColumnHolder unexistingColumn : consistencyChecks.getUnexisingColumns()) { + keysToRemoveIds.addAll(tapSchemaDBBroker.getKeysToRemoveFromUnexistingColumn(tapSchemaName, unexistingColumn)); + } + for (KeyHolder unexistingKey : consistencyChecks.getUnexistingKeys()) { + keysToRemoveIds.add(unexistingKey.getKeyId()); + } + + tapSchemaDBBroker.deleteUnexistingEntities(tapSchemaName, consistencyChecks, keysToRemoveIds); + } + + private void addUnaddedTables() throws SQLException { + for (Map.Entry<String, Set<String>> entry : consistencyChecks.getTablesToAdd().entrySet()) { + String schemaName = entry.getKey(); + Schema schema = tapSchema.getChild(schemaName); + for (String tableName : entry.getValue()) { + schema.addChild(tableName); + } + } + } + + private void addUnaddedColumns() throws SQLException { + for (ColumnHolder columnHolder : consistencyChecks.getColumnsToAdd()) { + Schema schema = tapSchema.getChild(columnHolder.getSchemaName()); + Table table = schema.getChild(columnHolder.getTableName()); + table.addChild(columnHolder.getColumnName()); + } + } + + private void fixInconsistentValues() throws SQLException { + for (InconsistentColumnProperty inconsistency : consistencyChecks.getInconsistencies()) { + tapSchema.getTapSchemaDBBroker().updateTapSchemaColumnValue(tapSchema.getName(), + inconsistency.getTableCompleteName(), inconsistency.getColumnName(), + inconsistency.getKey(), inconsistency.getCorrectValue()); + } + } +} diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBroker.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBroker.java index 558df20f3b64948815e885c834e41990ab5253c2..bd8f8abc217fccc337ec943d5ff26ddd4ee3b7cd 100644 --- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBroker.java +++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBroker.java @@ -22,15 +22,19 @@ */ package it.inaf.ia2.tsm.datalayer; +import it.inaf.ia2.tsm.ColumnHolder; +import it.inaf.ia2.tsm.ConsistencyChecks; import it.inaf.ia2.tsm.Key; import it.inaf.ia2.tsm.TapSchema; import it.inaf.ia2.tsm.TapSchemaEntity; +import it.inaf.ia2.tsm.model.ColumnModel; import it.inaf.ia2.tsm.model.TableModel; import it.inaf.ia2.tsm.model.SchemaModel; import java.sql.Connection; import java.sql.SQLException; import java.util.List; import java.util.Map; +import java.util.Set; /** * @@ -67,6 +71,22 @@ public interface DBBroker { void createTapSchemaStructure(String tapSchemaName, SchemaModel tapSchemaModel) throws SQLException; void createIvoaSchemaStructure(SchemaModel ivoaSchemaModel) throws SQLException; - + void save(TapSchema tapSchema) throws SQLException; + + void createTable(String schemaName, TableModel tableModel) throws SQLException; + + void addColumn(ColumnHolder columnHolder, ColumnModel columnModel) throws SQLException; + + /** + * @param like is for using both the schema name and the complete table name + * in the query + */ + Set<String> getKeysToRemove(String tapSchemaName, String like) throws SQLException; + + void deleteUnexistingEntities(String tapSchemaName, ConsistencyChecks consistencyChecks, Set<String> keysToRemoveIds) throws SQLException; + + Set<String> getKeysToRemoveFromUnexistingColumn(String tapSchemaName, ColumnHolder unexistingColumn) throws SQLException; + + void updateTapSchemaColumnValue(String tapSchemaName, String completeTableName, String columnName, String key, Object value) throws SQLException; } diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBrokerTemplate.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBrokerTemplate.java index 7230bef246bfd3a052eaac41b7e05dc1d9e6e981..538e7fd35370f5711967b763a40ef1a168d0c4e9 100644 --- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBrokerTemplate.java +++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBrokerTemplate.java @@ -23,6 +23,8 @@ package it.inaf.ia2.tsm.datalayer; import it.inaf.ia2.tsm.Column; +import it.inaf.ia2.tsm.ColumnHolder; +import it.inaf.ia2.tsm.ConsistencyChecks; import it.inaf.ia2.tsm.Key; import it.inaf.ia2.tsm.KeyColumn; import it.inaf.ia2.tsm.Schema; @@ -46,8 +48,10 @@ import java.sql.Types; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import javax.sql.DataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -112,6 +116,13 @@ public abstract class DBBrokerTemplate implements DBBroker { sb.append(String.format("(%s)", size)); } + @Override + public void createTable(String schemaName, TableModel tableModel) throws SQLException { + try (Connection conn = dataSource.getConnection()) { + createTable(schemaName, tableModel, conn); + } + } + protected abstract void createTable(String schemaName, TableModel tableModel, Connection conn) throws SQLException; protected abstract String getAddPrimaryKeyQuery(String tapSchemaName, String tableName, String[] keyColumns); @@ -790,4 +801,208 @@ public abstract class DBBrokerTemplate implements DBBroker { protected DataTypeMode getDataTypeMode() { return dataTypeMode; } + + protected abstract String getDataTypeFromADQLType(String adqlType); + + @Override + public void addColumn(ColumnHolder columnHolder, ColumnModel columnModel) throws SQLException { + + String dataType = getDataTypeFromADQLType(columnModel.getType()); + if (columnModel.getSize() != null) { + dataType = String.format("%s(%s)", dataType, columnModel.getSize()); + } + + String query = String.format("ALTER TABLE %s.%s ADD COLUMN %s %s", + escape(columnHolder.getSchemaName()), escape(columnHolder.getTableName()), + escape(columnHolder.getColumnName()), dataType); + + LOG.debug("Executing query " + query); + + try (Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + statement.executeUpdate(query); + } + } + + @Override + public Set<String> getKeysToRemove(String tapSchemaName, String like) throws SQLException { + + Set<String> keysId = new HashSet<>(); + + String query = String.format("SELECT key_id from %s.%s WHERE from_table LIKE ? OR target_table LIKE ?", + escape(tapSchemaName), escape(TapSchema.KEYS_TABLE)); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement ps = connection.prepareStatement(query)) { + ps.setString(1, like + "%"); + ps.setString(2, like + "%"); + LOG.debug("Executing query: {} [{}]", query, like); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + keysId.add(rs.getString("key_id")); + } + } + } + + return keysId; + } + + @Override + public Set<String> getKeysToRemoveFromUnexistingColumn(String tapSchemaName, ColumnHolder unexistingColumn) throws SQLException { + + Set<String> keysToRemoveIds = new HashSet<>(); + + StringBuilder sb = new StringBuilder(); + sb.append("SELECT k.key_id AS key_id\n"); + sb.append("FROM "); + sb.append(tapSchemaName); + sb.append(".`keys` k\n"); + sb.append("JOIN "); + sb.append(tapSchemaName); + sb.append(".key_columns c ON k.key_id = c.key_id\n"); + sb.append("WHERE (k.from_table = ? AND c.from_column = ?) OR (k.target_table = ? AND c.target_column = ?)"); + + String query = sb.toString(); + + String completeTableName = String.format("%s.%s", + unexistingColumn.getSchemaName(), unexistingColumn.getTableName()); + + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(query)) { + + ps.setString(1, completeTableName); + ps.setString(2, unexistingColumn.getColumnName()); + ps.setString(3, completeTableName); + ps.setString(4, unexistingColumn.getColumnName()); + + LOG.debug("Executing query {}", query); + + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + keysToRemoveIds.add(rs.getString("key_id")); + } + } + } + + return keysToRemoveIds; + } + + @Override + public void deleteUnexistingEntities(String tapSchemaName, ConsistencyChecks consistencyChecks, Set<String> keysToRemoveIds) throws SQLException { + try (Connection conn = dataSource.getConnection()) { + + String query, tapSchemaNameEscaped = escape(tapSchemaName); + conn.setAutoCommit(false); + LOG.debug("Starting transaction"); + + try { + // Removing all key_columns + for (String keyId : keysToRemoveIds) { + query = String.format("DELETE FROM %s.%s WHERE key_id = ?", tapSchemaNameEscaped, escape(TapSchema.KEY_COLUMNS_TABLE)); + try (PreparedStatement ps = conn.prepareStatement(query)) { + ps.setString(1, keyId); + LOG.debug("Executing query {} [{}]", query, keyId); + ps.executeUpdate(); + } + } + + // Removing all keys + for (String keyId : keysToRemoveIds) { + query = String.format("DELETE FROM %s.%s WHERE key_id = ?", tapSchemaNameEscaped, escape(TapSchema.KEYS_TABLE)); + try (PreparedStatement ps = conn.prepareStatement(query)) { + ps.setString(1, keyId); + LOG.debug("Executing query {} [{}]", query, keyId); + ps.executeUpdate(); + } + } + + // Removing all columns + for (ColumnHolder unexistingColumn : consistencyChecks.getUnexisingColumns()) { + query = String.format("DELETE FROM %s.%s WHERE table_name = ? AND column_name = ?", tapSchemaNameEscaped, escape(TapSchema.COLUMNS_TABLE)); + try (PreparedStatement ps = conn.prepareStatement(query)) { + String completeTableName = String.format("%s.%s", + unexistingColumn.getSchemaName(), unexistingColumn.getTableName()); + ps.setString(1, completeTableName); + ps.setString(2, unexistingColumn.getColumnName()); + LOG.debug("Executing query {} [{}, {}]", query, completeTableName, unexistingColumn.getColumnName()); + ps.executeUpdate(); + } + } + for (String table : consistencyChecks.getUnexisingTables()) { + query = String.format("DELETE FROM %s.%s WHERE table_name = ?", tapSchemaNameEscaped, escape(TapSchema.COLUMNS_TABLE)); + try (PreparedStatement ps = conn.prepareStatement(query)) { + ps.setString(1, table); + LOG.debug("Executing query {} [{}]", query, table); + ps.executeUpdate(); + } + } + for (String schema : consistencyChecks.getUnexisingSchemas()) { + query = String.format("DELETE FROM %s.%s WHERE table_name LIKE ?", tapSchemaNameEscaped, escape(TapSchema.COLUMNS_TABLE)); + try (PreparedStatement ps = conn.prepareStatement(query)) { + ps.setString(1, schema + "%"); + LOG.debug("Executing query {} [{}%]", query, schema); + ps.executeUpdate(); + } + } + + // Removing all tables + for (String table : consistencyChecks.getUnexisingTables()) { + query = String.format("DELETE FROM %s.%s WHERE table_name = ?", tapSchemaNameEscaped, escape(TapSchema.TABLES_TABLE)); + try (PreparedStatement ps = conn.prepareStatement(query)) { + ps.setString(1, table); + LOG.debug("Executing query {} [{}]", query, table); + ps.executeUpdate(); + } + } + for (String schema : consistencyChecks.getUnexisingSchemas()) { + query = String.format("DELETE FROM %s.%s WHERE schema_name = ?", tapSchemaNameEscaped, escape(TapSchema.TABLES_TABLE)); + try (PreparedStatement ps = conn.prepareStatement(query)) { + ps.setString(1, schema); + LOG.debug("Executing query {} [{}]", query, schema); + ps.executeUpdate(); + } + } + + // Removing all schemas + for (String schema : consistencyChecks.getUnexisingSchemas()) { + query = String.format("DELETE FROM %s.%s WHERE schema_name = ?", tapSchemaNameEscaped, escape(TapSchema.SCHEMAS_TABLE)); + try (PreparedStatement ps = conn.prepareStatement(query)) { + ps.setString(1, schema); + LOG.debug("Executing query {} [{}]", query, schema); + ps.executeUpdate(); + } + } + + conn.commit(); + } catch (SQLException e) { + LOG.error("Exception detected. Executing rollback!", e); + try { + conn.rollback(); + conn.setAutoCommit(true); + } catch (SQLException er) { + LOG.error("Exception during rollback", er); + throw er; + } + } + } + } + + @Override + public void updateTapSchemaColumnValue(String tapSchemaName, String completeTableName, String columnName, String key, Object value) throws SQLException { + + String query = String.format("UPDATE %s.%s SET %s = ? WHERE table_name = ? and column_name = ?", + escape(tapSchemaName), escape(TapSchema.COLUMNS_TABLE), escape(key)); + + try (Connection conn = dataSource.getConnection(); + PreparedStatement ps = conn.prepareStatement(query)) { + + LOG.debug("Executing query {} [{}, {}, {}]", query, value, completeTableName, columnName); + + ps.setObject(1, value); + ps.setString(2, completeTableName); + ps.setString(3, columnName); + + ps.executeUpdate(); + } + } } diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/mysql/MySQLDBBroker.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/mysql/MySQLDBBroker.java index 455cc6b8001b57ed2ef637b96a852b4b5c948a4f..4c0443bd441fb819f9e6ce00fb198d9e8f6f1ba6 100644 --- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/mysql/MySQLDBBroker.java +++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/mysql/MySQLDBBroker.java @@ -55,6 +55,11 @@ public class MySQLDBBroker extends DBBrokerTemplate { super(dataSource, '`', mode); } + @Override + protected String getDataTypeFromADQLType(String adqlType) { + return TypesMapping.getMySQLTypeFromADQLType(adqlType); + } + @Override protected String getAllSchemaNamesQuery() { return "SHOW DATABASES"; diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/pgsql/PostgresDBBroker.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/pgsql/PostgresDBBroker.java index 35882b928b2eacc1ef0e2b185d6b864a92bacecf..81236c0ba0fa8ed491f55ed16ca791308a8bf9a7 100644 --- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/pgsql/PostgresDBBroker.java +++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/pgsql/PostgresDBBroker.java @@ -59,6 +59,11 @@ public class PostgresDBBroker extends DBBrokerTemplate { this.pgDatabaseName = pgDatabaseName; } + @Override + protected String getDataTypeFromADQLType(String adqlType) { + return TypesMapping.getPostgresSQLTypeFromADQLType(adqlType); + } + @Override protected void createTable(String schemaName, TableModel tableModel, Connection conn) throws SQLException { StringBuilder querySb = new StringBuilder(); diff --git a/TASMAN-core/src/main/resources/schema_definition/tap_schema-1.xml b/TASMAN-core/src/main/resources/schema_definition/tap_schema-1.xml index e27dd723aac27681e26e0231dcfb327e370aa352..07dc153b07bbe611479626887336e5fb64bc3860 100644 --- a/TASMAN-core/src/main/resources/schema_definition/tap_schema-1.xml +++ b/TASMAN-core/src/main/resources/schema_definition/tap_schema-1.xml @@ -40,8 +40,7 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. <standard>true</standard> </column> <column name="description"> - <type>VARCHAR</type> - <size>255</size> + <type>CLOB</type> <updatable>true</updatable> <description>describes schemas in the tableset</description> <standard>true</standard> @@ -82,8 +81,7 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. <standard>true</standard> </column> <column name="description"> - <type>VARCHAR</type> - <size>255</size> + <type>CLOB</type> <updatable>true</updatable> <description>describes tables in the tableset</description> <standard>true</standard> @@ -126,8 +124,7 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. <standard>true</standard> </column> <column name="description"> - <type>VARCHAR</type> - <size>255</size> + <type>CLOB</type> <updatable>true</updatable> <description>describes the columns in the tableset</description> <standard>true</standard> @@ -204,8 +201,7 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. <standard>true</standard> </column> <column name="description"> - <type>VARCHAR</type> - <size>255</size> + <type>CLOB</type> <updatable>true</updatable> <description>describes keys in the tableset</description> <standard>true</standard> diff --git a/TASMAN-core/src/test/java/it/inaf/ia2/tsm/TestAll.java b/TASMAN-core/src/test/java/it/inaf/ia2/tsm/TestAll.java index 62920a0997fe1dcd578df875ad9d1589409a511a..ee2b2b218552ad4092ed180627cc3eb2d08fb44e 100644 --- a/TASMAN-core/src/test/java/it/inaf/ia2/tsm/TestAll.java +++ b/TASMAN-core/src/test/java/it/inaf/ia2/tsm/TestAll.java @@ -460,7 +460,7 @@ public class TestAll { assertFalse(operations.getHasOperations()); // Checking size and arraysize detection - Column descriptionColumn = tapSchema.getChild(tapSchema.getName()).getChild("columns").getChild("description"); + Column descriptionColumn = tapSchema.getChild(tapSchema.getName()).getChild("columns").getChild("utype"); assertEquals("char", descriptionColumn.getProperty("datatype").getValue(String.class)); assertEquals(255, descriptionColumn.getProperty("size").getValue(Integer.class)); assertEquals("255*", descriptionColumn.getProperty("arraysize").getValue(String.class)); diff --git a/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/ConsistencyChecksBean.java b/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/ConsistencyChecksBean.java index f44f84d09cda1bebdbfe5630e61dcbfe0346657d..ec52620d48346d59113d7f9a447d0c22dbcb0784 100644 --- a/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/ConsistencyChecksBean.java +++ b/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/ConsistencyChecksBean.java @@ -23,7 +23,7 @@ package it.inaf.ia2.tsm.webapp; import it.inaf.ia2.tsm.TapSchema; -import it.inaf.ia2.tsm.datalayer.DBWrapper; +import it.inaf.ia2.tsm.TapSchemaMender; import java.io.Serializable; import java.sql.SQLException; import javax.inject.Inject; @@ -43,13 +43,8 @@ public class ConsistencyChecksBean implements Serializable { @Inject TapSchemaEditingBean tapSchemaEditingBean; - private DBWrapper dbWrapper; private TapSchema tapSchema; - public void setDbWrapper(DBWrapper dbWrapper) { - this.dbWrapper = dbWrapper; - } - public TapSchema getTapSchema() { return tapSchema; } @@ -64,7 +59,7 @@ public class ConsistencyChecksBean implements Serializable { } public String proceed() throws SQLException { - tapSchema.getConsistencyChecks().amendTapSchema(dbWrapper, tapSchema); + TapSchemaMender.amendTapSchema(tapSchema); tapSchema.save(); tapSchemaEditingBean.setTapSchema(tapSchema); return "tapSchemaEditing.xhtml?faces-redirect=true"; diff --git a/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/TapSchemaLoader.java b/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/TapSchemaLoader.java index 46a1b26b684012fd78c84505a1cf004a4558cbe7..0c3559d140388ec26cb2aa14cdb3694880aee09a 100644 --- a/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/TapSchemaLoader.java +++ b/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/TapSchemaLoader.java @@ -175,7 +175,6 @@ public class TapSchemaLoader implements Serializable { public String openLoaded() { if (loadedTapSchema.getConsistencyChecks().isInconsistent()) { - consistencyChecksBean.setDbWrapper(dbWrapper); consistencyChecksBean.setTapSchema(loadedTapSchema); return "consistencyChecks.xhtml?faces-redirect=true"; } else { diff --git a/TASMAN-webapp/src/main/webapp/consistencyChecks.xhtml b/TASMAN-webapp/src/main/webapp/consistencyChecks.xhtml index f2d9580318e44c5e8e4a7deed444a1bc729dc7a1..80e28b2bed7c75f8978d8aa3024d860e9df4566e 100644 --- a/TASMAN-webapp/src/main/webapp/consistencyChecks.xhtml +++ b/TASMAN-webapp/src/main/webapp/consistencyChecks.xhtml @@ -19,9 +19,9 @@ <table class="table table-bordered"> <thead> <tr> - <th>TAP_SCHEMA entity</th> - <th>Name/Description</th> - <th>Wrong field</th> + <th>Table</th> + <th>Column</th> + <th>Field</th> <th>Wrong value</th> <th>Correct value</th> </tr> @@ -29,9 +29,9 @@ <tbody> <ui:repeat value="#{consistency.tapSchema.consistencyChecks.inconsistencies}" var="inconsistency"> <tr> - <td>${inconsistency.tapSchemaEntityType}</td> - <td>${inconsistency.tapSchemaEntityDescription}</td> - <td>${inconsistency.wrongPropertyName}</td> + <td>${inconsistency.tableCompleteName}</td> + <td>${inconsistency.columnName}</td> + <td>${inconsistency.key}</td> <td class="text-danger">${inconsistency.currentValue}</td> <td class="text-success">${inconsistency.correctValue}</td> </tr> @@ -41,7 +41,7 @@ </h:panelGroup> <h:panelGroup rendered="#{consistency.tapSchema.consistencyChecks.unexisingSchemas.size() gt 0}"> - <h2>Missing schemas</h2> + <h2>Unexisting schemas</h2> <ul> <ui:repeat value="#{consistency.tapSchema.consistencyChecks.unexisingSchemas}" var="schema"> <li>${schema}</li> @@ -50,7 +50,7 @@ </h:panelGroup> <h:panelGroup rendered="#{consistency.tapSchema.consistencyChecks.unexisingTables.size() gt 0}"> - <h2>Missing tables</h2> + <h2>Unexisting tables</h2> <ul> <ui:repeat value="#{consistency.tapSchema.consistencyChecks.unexisingTables}" var="table"> <li>${table}</li> @@ -59,29 +59,74 @@ </h:panelGroup> <h:panelGroup rendered="#{consistency.tapSchema.consistencyChecks.unexisingColumns.size() gt 0}"> - <h2>Missing columns</h2> + <h2>Unexisting columns</h2> <ul> - <ui:repeat value="#{consistency.tapSchema.consistencyChecks.unexisingColumns}" var="column"> + <ui:repeat value="#{consistency.tapSchema.consistencyChecks.unexisingColumns.toArray()}" var="column"> <li>${column}</li> </ui:repeat> </ul> </h:panelGroup> - + <h:panelGroup rendered="#{consistency.tapSchema.consistencyChecks.unexistingKeys.size() gt 0}"> - <h2>Missing keys</h2> + <h2>Unexisting keys</h2> <ul> - <ui:repeat value="#{consistency.tapSchema.consistencyChecks.unexistingKeys}" var="key"> + <ui:repeat value="#{consistency.tapSchema.consistencyChecks.unexistingKeys.toArray()}" var="key"> <li>${key}</li> </ui:repeat> </ul> </h:panelGroup> + + <h:panelGroup rendered="#{consistency.tapSchema.consistencyChecks.missingTables.size() gt 0}"> + <h2>Missing tables</h2> + <ul> + <ui:repeat value="#{consistency.tapSchema.consistencyChecks.missingTables.keySet().toArray()}" var="schema"> + <ui:repeat value="#{consistency.tapSchema.consistencyChecks.missingTables.get(schema).toArray()}" var="table"> + <li>${schema}.${table}</li> + </ui:repeat> + </ui:repeat> + </ul> + </h:panelGroup> + + <h:panelGroup rendered="#{consistency.tapSchema.consistencyChecks.missingColumns.size() gt 0}"> + <h2>Missing columns</h2> + <ul> + <ui:repeat value="#{consistency.tapSchema.consistencyChecks.missingColumns.keySet().toArray()}" var="column"> + <li>${column}</li> + </ui:repeat> + </ul> + </h:panelGroup> + + <h:panelGroup rendered="#{consistency.tapSchema.consistencyChecks.tablesToAdd.size() gt 0}"> + <h2>Tables to expose</h2> + <ul> + <ui:repeat value="#{consistency.tapSchema.consistencyChecks.tablesToAdd.keySet().toArray()}" var="schema"> + <ui:repeat value="#{consistency.tapSchema.consistencyChecks.tablesToAdd.get(schema)}" var="table"> + <li>${schema}.${table}</li> + </ui:repeat> + </ui:repeat> + </ul> + </h:panelGroup> + + <h:panelGroup rendered="#{consistency.tapSchema.consistencyChecks.columnsToAdd.size() gt 0}"> + <h2>Columns to expose</h2> + <ul> + <ui:repeat value="#{consistency.tapSchema.consistencyChecks.columnsToAdd.toArray()}" var="column"> + <li>${column}</li> + </ui:repeat> + </ul> + </h:panelGroup> + + <h:panelGroup rendered="#{consistency.tapSchema.consistencyChecks.missingObscore or consistency.tapSchema.consistencyChecks.obscoreToAdd}"> + <h2>ObsCore to add</h2> + <span class="glyphicon glyphicon-check"></span> + </h:panelGroup> <br/> <h:panelGroup rendered="#{consistency.tapSchemaContainsOnlyTapSchema}" layout="block" class="alert alert-danger text-center"> <span class="glyphicon glyphicon-warning-sign"></span> <strong>If you proceed this TAP_SCHEMA will contain only itself! You may have selected wrong source credentials.</strong> </h:panelGroup> - + <div class="alert alert-warning text-center"> <span class="glyphicon glyphicon-warning-sign"></span> <strong>If you proceed the TAP_SCHEMA Manager will fix these values.</strong>