From 243decdb7903fb61e93180796de02aab679b17fb Mon Sep 17 00:00:00 2001
From: Sonia Zorba <zorba@oats.inaf.it>
Date: Tue, 29 Aug 2017 16:53:33 +0200
Subject: [PATCH] Consistency checking improvements
---
.../src/main/java/it/inaf/ia2/tsm/Column.java | 11 ++-
.../java/it/inaf/ia2/tsm/ColumnHolder.java | 7 +-
.../it/inaf/ia2/tsm/ConsistencyChecks.java | 41 ++++++++-
.../ia2/tsm/InconsistentColumnProperty.java | 22 +++--
.../src/main/java/it/inaf/ia2/tsm/Table.java | 18 +++-
.../java/it/inaf/ia2/tsm/TapSchemaLoader.java | 33 ++++---
.../java/it/inaf/ia2/tsm/TapSchemaMender.java | 21 +++--
.../java/it/inaf/ia2/tsm/WrongDataType.java | 91 +++++++++++++++++++
.../it/inaf/ia2/tsm/datalayer/DBBroker.java | 4 +-
.../ia2/tsm/datalayer/DBBrokerTemplate.java | 53 ++++++++++-
.../tsm/datalayer/mysql/MySQLDBBroker.java | 38 +++++++-
.../tsm/datalayer/pgsql/PostgresDBBroker.java | 79 +++++++++++++++-
.../it/inaf/ia2/tsm/model/ColumnModel.java | 33 +++++++
.../tsm/model/CommaSeparatedListAdapter.java | 59 ++++++++++++
.../it/inaf/ia2/tsm/model/TypesMapping.java | 20 ++++
.../resources/schema_definition/ivoa-1_1.xml | 34 +------
.../schema_definition/tap_schema-1.xml | 4 -
.../schema_definition/tap_schema-1_1.xml | 2 -
.../test/java/it/inaf/ia2/tsm/TestAll.java | 6 --
.../ia2/tsm/webapp/ConsistencyChecksBean.java | 10 +-
.../ia2/tsm/webapp/TapSchemaEditingBean.java | 12 +++
.../inaf/ia2/tsm/webapp/TapSchemaLoader.java | 12 ++-
.../src/main/webapp/consistencyChecks.xhtml | 48 ++++++++--
.../src/main/webapp/credentialsEditing.xhtml | 6 +-
.../main/webapp/resources/js/async-loader.js | 3 +
.../src/main/webapp/tapSchemaEditing.xhtml | 10 +-
26 files changed, 569 insertions(+), 108 deletions(-)
create mode 100644 TASMAN-core/src/main/java/it/inaf/ia2/tsm/WrongDataType.java
create mode 100644 TASMAN-core/src/main/java/it/inaf/ia2/tsm/model/CommaSeparatedListAdapter.java
diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/Column.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/Column.java
index 96cf36c..dd168a0 100644
--- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/Column.java
+++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/Column.java
@@ -44,6 +44,9 @@ public class Column extends ChildEntity<Table> {
public final static String UCD_KEY = "ucd";
public final static String UNIT_KEY = "unit";
+ // Original datatype (computed from information_schema data), used for consistency checking
+ public final static String ORIGINAL_DATATYPE_KEY = "original_datatype";
+
private static final long serialVersionUID = 9175956487892235521L;
private static final Logger LOG = LoggerFactory.getLogger(Column.class);
@@ -51,15 +54,17 @@ public class Column extends ChildEntity<Table> {
private Key foreignKey;
private Table parentTable;
+ private boolean mandatory;
private Column() {
// for serialization
super();
}
- protected Column(TapSchema tapSchema, Table table, String columnName) {
+ protected Column(TapSchema tapSchema, Table table, String columnName, boolean mandatory) {
super(tapSchema, tapSchema.getTableModel(TapSchema.COLUMNS_TABLE), table.getColumnMetadata(columnName));
parentTable = table;
+ this.mandatory = mandatory;
setStatus(Status.LOADED);
}
@@ -114,6 +119,10 @@ public class Column extends ChildEntity<Table> {
return (boolean) getMetadata(PRIMARY_KEY);
}
+ public boolean isMandatory() {
+ return mandatory;
+ }
+
@Override
public int hashCode() {
int hash = 7;
diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/ColumnHolder.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/ColumnHolder.java
index 8f782bb..57a870d 100644
--- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/ColumnHolder.java
+++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/ColumnHolder.java
@@ -24,7 +24,6 @@ package it.inaf.ia2.tsm;
import java.io.Serializable;
import java.util.Objects;
-import java.util.regex.Pattern;
/**
*
@@ -47,6 +46,12 @@ public class ColumnHolder implements Serializable {
this.columnName = columnName;
}
+ public ColumnHolder(Column column) {
+ this.schemaName = column.getParent().getParent().getName();
+ this.tableName = column.getParent().getName();
+ this.columnName = column.getName();
+ }
+
public String getSchemaName() {
return schemaName;
}
diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/ConsistencyChecks.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/ConsistencyChecks.java
index 74aa204..7b34732 100644
--- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/ConsistencyChecks.java
+++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/ConsistencyChecks.java
@@ -53,9 +53,13 @@ public class ConsistencyChecks implements Serializable {
private final Map<String, Set<String>> tablesToAdd;
private final Map<ColumnHolder, ColumnModel> missingColumns;
private final Set<ColumnHolder> columnsToAdd;
+ private final List<WrongDataType> wrongDataTypes;
private boolean missingObscore;
private boolean obscoreToAdd;
+ // This will not consider an inconsistency: it is only used to display a warning
+ private final Set<ColumnHolder> unaddedOptionalColumns;
+
public ConsistencyChecks() {
inconsistencies = new ArrayList<>();
unexisingSchemas = new HashSet<>();
@@ -66,6 +70,8 @@ public class ConsistencyChecks implements Serializable {
tablesToAdd = new HashMap<>();
missingColumns = new HashMap<>();
columnsToAdd = new HashSet<>();
+ unaddedOptionalColumns = new HashSet<>();
+ wrongDataTypes = new ArrayList<>();
}
public void addInconsistency(InconsistentColumnProperty problemDescription) {
@@ -182,11 +188,44 @@ public class ConsistencyChecks implements Serializable {
this.obscoreToAdd = obscoreToAdd;
}
+ public void addUnaddedOptionalColumn(ColumnHolder columnHolder) {
+ unaddedOptionalColumns.add(columnHolder);
+ }
+
+ public Set<ColumnHolder> getUnaddedOptionalColumns() {
+ return unaddedOptionalColumns;
+ }
+
+ public void addWrongDataType(ColumnHolder columnHolder, String wrongDataType, String correctDataType, String adqlCorrectDataType, Integer size) {
+ // If datatype needs to be changed inconsistency on it doesn't make sense anymore.
+ Iterator<InconsistentColumnProperty> ite = inconsistencies.iterator();
+ while (ite.hasNext()) {
+ InconsistentColumnProperty inconsistency = ite.next();
+ if (inconsistency.getColumnHolder().equals(columnHolder)
+ && (inconsistency.getKey().equals(Column.DATATYPE_KEY)
+ || inconsistency.getKey().equals(Column.SIZE_KEY)
+ || inconsistency.getKey().equals(Column.ARRAYSIZE_KEY))) {
+ ite.remove();
+ }
+ }
+
+ WrongDataType wdt = new WrongDataType(columnHolder, wrongDataType, correctDataType, adqlCorrectDataType, size);
+ wrongDataTypes.add(wdt);
+ }
+
+ public List<WrongDataType> getWrongDataTypes() {
+ return wrongDataTypes;
+ }
+
public boolean isInconsistent() {
- return !inconsistencies.isEmpty()
+ return !inconsistencies.isEmpty() || !wrongDataTypes.isEmpty()
|| !unexisingSchemas.isEmpty() || !unexisingTables.isEmpty() || !unexistingColumns.isEmpty()
|| !missingTables.isEmpty() || !missingColumns.isEmpty()
|| !columnsToAdd.isEmpty() || !tablesToAdd.isEmpty()
|| obscoreToAdd || missingObscore;
}
+
+ public boolean isHasWarnings() {
+ return !unaddedOptionalColumns.isEmpty();
+ }
}
diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentColumnProperty.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentColumnProperty.java
index e368bed..fc567ab 100644
--- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentColumnProperty.java
+++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentColumnProperty.java
@@ -32,8 +32,7 @@ public class InconsistentColumnProperty implements Serializable {
private static final long serialVersionUID = -5145865322582594970L;
- private String tableCompleteName;
- private String columnName;
+ private ColumnHolder columnHolder;
private String key;
private Object currentValue;
private Object correctValue;
@@ -41,22 +40,29 @@ public class InconsistentColumnProperty implements Serializable {
private InconsistentColumnProperty() {
}
- public InconsistentColumnProperty(String tableCompleteName, String columnName, String key, Object currentValue, Object correctValue) {
- this.tableCompleteName = tableCompleteName;
- this.columnName = columnName;
+ public InconsistentColumnProperty(ColumnHolder columnHolder, String key, Object currentValue, Object correctValue) {
+ this.columnHolder = columnHolder;
this.key = key;
this.currentValue = currentValue;
this.correctValue = correctValue;
}
+ public ColumnHolder getColumnHolder() {
+ return columnHolder;
+ }
+
+ public void setColumnHolder(ColumnHolder columnHolder) {
+ this.columnHolder = columnHolder;
+ }
+
public String getTableCompleteName() {
- return tableCompleteName;
+ return String.format("%s.%s", columnHolder.getSchemaName(), columnHolder.getTableName());
}
public String getColumnName() {
- return columnName;
+ return columnHolder.getColumnName();
}
-
+
public String getKey() {
return key;
}
diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/Table.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/Table.java
index a5a4a83..fb8388d 100644
--- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/Table.java
+++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/Table.java
@@ -52,12 +52,15 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu
private String simpleName;
private Schema parentSchema;
+ // WARNING: this is different from tableModel inherited field.
+ private TableModel tableTableModel;
+
private Table() {
// for serialization
super();
}
- private TableModel getModel() {
+ private TableModel getTableTableModel() {
if (tapSchema.getName().equals(parentSchema.getName())) {
return tapSchema.getTapSchemaModel().getTable(simpleName);
}
@@ -77,7 +80,9 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu
columns = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
DBBroker broker = tapSchema.getDBBroker(schema.getName());
- columnsMetadata = broker.getAllColumnsMetadata(schema.getName(), tableSimpleName, getModel(), tapSchema.getVersion());
+ tableTableModel = getTableTableModel();
+ columnsMetadata = broker.getAllColumnsMetadata(schema.getName(), tableSimpleName, tableTableModel, tapSchema.getDataTypeMode());
+
for (Map.Entry<String, Map<String, Object>> entry : columnsMetadata.entrySet()) {
// Adding table names to columns metadata
entry.getValue().put(Column.TABLE_NAME_KEY, schema.getName() + "." + tableSimpleName);
@@ -103,6 +108,13 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu
return getValue(TABLE_NAME_KEY, String.class);
}
+ private boolean isMandatory(String columnName) {
+ if (tableTableModel != null) {
+ return tableTableModel.get(columnName).isMandatory();
+ }
+ return false;
+ }
+
/**
* {@inheritDoc }
*/
@@ -117,7 +129,7 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu
} else {
Column column = columns.get(columnName);
if (column == null) {
- column = new Column(tapSchema, this, columnName);
+ column = new Column(tapSchema, this, columnName, isMandatory(columnName));
columns.put(columnName, column);
column.setStatus(Status.ADDED_NOT_PERSISTED);
} else {
diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchemaLoader.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchemaLoader.java
index beebaec..2f75ae8 100644
--- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchemaLoader.java
+++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchemaLoader.java
@@ -28,10 +28,10 @@ import static it.inaf.ia2.tsm.TapSchema.KEY_COLUMNS_TABLE;
import static it.inaf.ia2.tsm.TapSchema.SCHEMAS_TABLE;
import static it.inaf.ia2.tsm.TapSchema.TABLES_TABLE;
import it.inaf.ia2.tsm.datalayer.DBBroker;
-import it.inaf.ia2.tsm.datalayer.DataTypeMode;
import it.inaf.ia2.tsm.model.ColumnModel;
import it.inaf.ia2.tsm.model.SchemaModel;
import it.inaf.ia2.tsm.model.TableModel;
+import it.inaf.ia2.tsm.model.TypeMapping;
import it.inaf.ia2.tsm.model.TypesMapping;
import java.sql.SQLException;
import java.util.ArrayList;
@@ -117,15 +117,27 @@ public class TapSchemaLoader {
Column column = table.getChild(columnModel.getName());
- if (columnModel.isMandatory() && column == null) {
+ ColumnHolder ch = new ColumnHolder(schemaName,
+ table.getName(), columnModel.getName());
- ColumnHolder ch = new ColumnHolder(schemaName,
- table.getName(), columnModel.getName());
-
- if (table.isAddable(columnModel.getName())) {
- consistencyChecks.addColumnToAdd(ch);
- } else {
- consistencyChecks.addMissingColumn(ch, columnModel);
+ if (column == null) {
+ if (columnModel.isMandatory()) {
+ if (table.isAddable(columnModel.getName())) {
+ consistencyChecks.addColumnToAdd(ch);
+ } else {
+ consistencyChecks.addMissingColumn(ch, columnModel);
+ }
+ } else if (table.isAddable(columnModel.getName())) {
+ consistencyChecks.addUnaddedOptionalColumn(ch);
+ }
+ } else {
+ // Data type checking
+ String originalDataType = (String) column.getMetadata(Column.ORIGINAL_DATATYPE_KEY);
+ String modelDataType = (String) column.getMetadata(Column.DATATYPE_KEY);
+ TypeMapping originalType = TypesMapping.getTypeMapping(originalDataType, tapSchema.getDataTypeMode());
+ TypeMapping modelType = TypesMapping.getTypeMapping(modelDataType, tapSchema.getDataTypeMode());
+ if (originalType.getJavaType() != modelType.getJavaType()) {
+ consistencyChecks.addWrongDataType(ch, originalDataType, modelDataType, columnModel.getType(), columnModel.getSize());
}
}
}
@@ -203,8 +215,7 @@ public class TapSchemaLoader {
Object correctValue = getCorrectValue(column, key);
if (!Objects.equals(savedValue, correctValue)) {
InconsistentColumnProperty inconsistentValue = new InconsistentColumnProperty(
- column.getTableCompleteName(),
- column.getName(),
+ new ColumnHolder(column),
key,
savedValue,
correctValue
diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchemaMender.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchemaMender.java
index 18f1152..9a8c424 100644
--- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchemaMender.java
+++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchemaMender.java
@@ -47,15 +47,7 @@ public class TapSchemaMender {
}
public static void amendTapSchema(TapSchema tapSchema) throws SQLException {
- int counter = 0;
- while (tapSchema.getConsistencyChecks() != null && tapSchema.getConsistencyChecks().isInconsistent()) {
- new TapSchemaMender(tapSchema).amendTapSchema();
- tapSchema.load();
- counter++;
- if (counter > 20) {
- throw new RuntimeException("Unable to amend TAP_SCHEMA");
- }
- }
+ new TapSchemaMender(tapSchema).amendTapSchema();
}
private void amendTapSchema() throws SQLException {
@@ -63,6 +55,7 @@ public class TapSchemaMender {
fixObscore();
createMissingTables();
createMissingColumns();
+ fixDataTypes();
deleteUnexistingEntities();
addUnaddedTables();
addUnaddedColumns();
@@ -107,6 +100,16 @@ public class TapSchemaMender {
}
}
+ private void fixDataTypes() throws SQLException {
+ for (WrongDataType wrongDataType : consistencyChecks.getWrongDataTypes()) {
+ ColumnHolder columnHolder = wrongDataType.getColumnHolder();
+ DBBroker broker = tapSchema.getDBBroker(columnHolder.getSchemaName());
+ broker.alterDataType(columnHolder.getSchemaName(),
+ columnHolder.getTableName(), columnHolder.getColumnName(),
+ wrongDataType.getAdqlCorrectDataType(), wrongDataType.getSize());
+ }
+ }
+
private void deleteUnexistingEntities() throws SQLException {
String tapSchemaName = tapSchema.getName();
DBBroker tapSchemaDBBroker = tapSchema.getTapSchemaDBBroker();
diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/WrongDataType.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/WrongDataType.java
new file mode 100644
index 0000000..cd429eb
--- /dev/null
+++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/WrongDataType.java
@@ -0,0 +1,91 @@
+/*
+ * _____________________________________________________________________________
+ *
+ * INAF - OATS National Institute for Astrophysics - Astronomical Observatory of
+ * Trieste INAF - IA2 Italian Center for Astronomical Archives
+ * _____________________________________________________________________________
+ *
+ * Copyright (C) 2017 Istituto Nazionale di Astrofisica
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License Version 3 as published by the
+ * Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package it.inaf.ia2.tsm;
+
+import java.io.Serializable;
+
+/**
+ *
+ * @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
+ */
+public class WrongDataType implements Serializable {
+
+ private static final long serialVersionUID = 1541867434766884291L;
+
+ private ColumnHolder columnHolder;
+ private String wrongDataType;
+ private String correctDataType;
+ private String adqlCorrectDataType;
+ private Integer size;
+
+ public WrongDataType() {
+ }
+
+ public WrongDataType(ColumnHolder columnHolder, String wrongDataType, String correctDataType, String adqlCorrectDataType, Integer size) {
+ this.columnHolder = columnHolder;
+ this.wrongDataType = wrongDataType;
+ this.correctDataType = correctDataType;
+ this.adqlCorrectDataType = adqlCorrectDataType;
+ this.size = size;
+ }
+
+ public ColumnHolder getColumnHolder() {
+ return columnHolder;
+ }
+
+ public void setColumnHolder(ColumnHolder columnHolder) {
+ this.columnHolder = columnHolder;
+ }
+
+ public String getWrongDataType() {
+ return wrongDataType;
+ }
+
+ public void setWrongDataType(String wrongDataType) {
+ this.wrongDataType = wrongDataType;
+ }
+
+ public String getCorrectDataType() {
+ return correctDataType;
+ }
+
+ public void setCorrectDataType(String correctDataType) {
+ this.correctDataType = correctDataType;
+ }
+
+ public String getAdqlCorrectDataType() {
+ return adqlCorrectDataType;
+ }
+
+ public void setAdqlCorrectDataType(String adqlCorrectDataType) {
+ this.adqlCorrectDataType = adqlCorrectDataType;
+ }
+
+ public Integer getSize() {
+ return size;
+ }
+
+ public void setSize(Integer size) {
+ this.size = size;
+ }
+}
diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBroker.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBroker.java
index bd8f8ab..e60eea6 100644
--- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBroker.java
+++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBroker.java
@@ -56,7 +56,7 @@ public interface DBBroker {
List<String> getAllColumnsNames(String schemaName, String tableName) throws SQLException;
- Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableSimpleName, TableModel tableModel, String tapSchemaVersion) throws SQLException;
+ Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableSimpleName, TableModel tableModel, DataTypeMode dataTypeMode) throws SQLException;
List<Key> getKeys(TapSchema tapSchema, String schemaName) throws SQLException;
@@ -89,4 +89,6 @@ public interface DBBroker {
Set<String> getKeysToRemoveFromUnexistingColumn(String tapSchemaName, ColumnHolder unexistingColumn) throws SQLException;
void updateTapSchemaColumnValue(String tapSchemaName, String completeTableName, String columnName, String key, Object value) throws SQLException;
+
+ void alterDataType(String schemaName, String tableName, String columnName, String adqlDataType, Integer size) throws SQLException;
}
diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBrokerTemplate.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBrokerTemplate.java
index 538e7fd..af964c8 100644
--- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBrokerTemplate.java
+++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBrokerTemplate.java
@@ -781,17 +781,25 @@ public abstract class DBBrokerTemplate implements DBBroker {
protected abstract Map<String, Map<String, Object>> getAllColumnsOriginalMetadata(String schemaName, String tableName) throws SQLException;
@Override
- public Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableName, TableModel tableModel, String tapSchemaVersion) throws SQLException {
+ public Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableName, TableModel tableModel, DataTypeMode dataTypeMode) throws SQLException {
Map<String, Map<String, Object>> metadata = getAllColumnsOriginalMetadata(schemaName, tableName);
- // Overriding data type from models
+ // Special behavior for data type
if (tableModel != null) {
for (Map.Entry<String, Map<String, Object>> entry : metadata.entrySet()) {
+
String columnName = entry.getKey();
+
+ // Saving original data type (used for consistency checking)
+ Map<String, Object> columnMetadata = entry.getValue();
+ String originalDataType = (String) columnMetadata.get(Column.DATATYPE_KEY);
+ columnMetadata.put(Column.ORIGINAL_DATATYPE_KEY, originalDataType);
+
+ // Override data type using model definition
String adqlType = tableModel.get(columnName).getType();
- String tapDataType = TypesMapping.getDataType(adqlType, dataTypeMode);
- entry.getValue().put(Column.DATATYPE_KEY, tapDataType);
+ String definedDataType = TypesMapping.getDataType(adqlType, dataTypeMode);
+ columnMetadata.put(Column.DATATYPE_KEY, definedDataType);
}
}
@@ -1005,4 +1013,41 @@ public abstract class DBBrokerTemplate implements DBBroker {
ps.executeUpdate();
}
}
+
+ protected String getEnumDefinition(List<String> enumValues) {
+ StringBuilder sb = new StringBuilder();
+
+ boolean first = true;
+ for (String enumValue : enumValues) {
+ if (!first) {
+ sb.append(',');
+ }
+ sb.append(String.format("'%s'", enumValue));
+ first = false;
+ }
+
+ return sb.toString();
+ }
+
+ private String getCheckConstraint(ColumnModel columnModel) {
+ if (columnModel.getMinValue() != null && columnModel.getMaxValue() != null) {
+ return String.format("CHECK (%s >= %s AND %s <= %s)", columnModel.getName(),
+ columnModel.getMinValue(), columnModel.getName(), columnModel.getMaxValue());
+ } else if (columnModel.getMinValue() != null && columnModel.getMaxValue() == null) {
+ return String.format("CHECK (%s >= %s)", columnModel.getName(), columnModel.getMinValue());
+ } else if (columnModel.getMinValue() == null && columnModel.getMaxValue() != null) {
+ return String.format("CHECK (%s <= %s)", columnModel.getName(), columnModel.getMaxValue());
+ }
+ return null;
+ }
+
+ protected void appendCheckConstraints(StringBuilder querySb, TableModel tableModel) {
+ for (ColumnModel cm : tableModel.getColumns()) {
+ String checkConstraint = getCheckConstraint(cm);
+ if (checkConstraint != null) {
+ querySb.append(",\n");
+ querySb.append(checkConstraint);
+ }
+ }
+ }
}
diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/mysql/MySQLDBBroker.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/mysql/MySQLDBBroker.java
index 4c0443b..816e002 100644
--- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/mysql/MySQLDBBroker.java
+++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/mysql/MySQLDBBroker.java
@@ -50,6 +50,7 @@ import org.slf4j.LoggerFactory;
public class MySQLDBBroker extends DBBrokerTemplate {
private static final Logger LOG = LoggerFactory.getLogger(MySQLDBBroker.class);
+ private static final int DEFAULT_VARCHAR_SIZE = 255;
public MySQLDBBroker(DataSource dataSource, DataTypeMode mode) {
super(dataSource, '`', mode);
@@ -170,10 +171,21 @@ public class MySQLDBBroker extends DBBrokerTemplate {
querySb.append(cm.getName());
querySb.append(" ");
- String mySQLType = TypesMapping.getMySQLTypeFromADQLType(cm.getType()).toUpperCase();
- querySb.append(mySQLType);
- if (mySQLType.equals("VARCHAR") || mySQLType.equals("CHAR")) {
- appendSize(querySb, cm.getSize());
+ String mySQLType;
+ if (cm.getEnumValues() == null) {
+ mySQLType = TypesMapping.getMySQLTypeFromADQLType(cm.getType()).toUpperCase();
+ querySb.append(mySQLType);
+ Integer size = cm.getSize();
+ if (mySQLType.equals("VARCHAR") && size == null) {
+ size = DEFAULT_VARCHAR_SIZE;
+ }
+ if (size != null) {
+ appendSize(querySb, size);
+ }
+ } else {
+ querySb.append("ENUM (");
+ querySb.append(getEnumDefinition(cm.getEnumValues()));
+ querySb.append(")");
}
if (cm.isNullable()) {
@@ -183,6 +195,7 @@ public class MySQLDBBroker extends DBBrokerTemplate {
}
}
}
+ appendCheckConstraints(querySb, tableModel);
querySb.append(")");
@@ -284,4 +297,21 @@ public class MySQLDBBroker extends DBBrokerTemplate {
protected String getTableTypesQuery(String schemaName) {
return "SELECT table_name, table_type FROM information_schema.tables WHERE table_schema = '" + schemaName + "'";
}
+
+ @Override
+ public void alterDataType(String schemaName, String tableName, String columnName, String adqlDataType, Integer size) throws SQLException {
+
+ String mySQLDataType = TypesMapping.getMySQLTypeFromADQLType(adqlDataType);
+ if (size != null) {
+ mySQLDataType = String.format("%s(%s)", mySQLDataType, size);
+ }
+
+ String query = String.format("ALTER TABLE %s.%s MODIFY %s %s", escape(schemaName), escape(tableName), escape(columnName), mySQLDataType);
+
+ try (Connection connection = dataSource.getConnection();
+ Statement statement = connection.createStatement()) {
+ LOG.debug("Executing query: {}", query);
+ statement.executeUpdate(query);
+ }
+ }
}
diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/pgsql/PostgresDBBroker.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/pgsql/PostgresDBBroker.java
index 81236c0..fe87f81 100644
--- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/pgsql/PostgresDBBroker.java
+++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/pgsql/PostgresDBBroker.java
@@ -32,6 +32,7 @@ import it.inaf.ia2.tsm.model.ColumnModel;
import it.inaf.ia2.tsm.model.TableModel;
import it.inaf.ia2.tsm.model.TypesMapping;
import java.sql.Connection;
+import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
@@ -64,8 +65,56 @@ public class PostgresDBBroker extends DBBrokerTemplate {
return TypesMapping.getPostgresSQLTypeFromADQLType(adqlType);
}
+ private boolean enumTypeExists(String enumName) throws SQLException {
+
+ String query = "SELECT 1 FROM pg_type WHERE typname = ?";
+
+ try (Connection conn = dataSource.getConnection();
+ PreparedStatement ps = conn.prepareStatement(query)) {
+ ps.setString(1, enumName);
+
+ LOG.debug("Executing query {}", query);
+ try (ResultSet rs = ps.executeQuery()) {
+ while (rs.next()) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private void createEnumType(String enumName, List<String> values) throws SQLException {
+
+ String query = String.format("CREATE TYPE %s AS ENUM (%s)", enumName, getEnumDefinition(values));
+
+ try (Connection conn = dataSource.getConnection();
+ Statement stat = conn.createStatement()) {
+
+ LOG.debug("Executing query {}", query);
+ stat.executeUpdate(query);
+ }
+ }
+
+ private String getEnumName(String schemaName, String tableName, String columnName) {
+ // WARNING: for some reasons enum names are stored ONLY lower case in Postgres
+ return String.format("TASMAN_%s_%s_%s", schemaName, tableName, columnName).toLowerCase();
+ }
+
@Override
protected void createTable(String schemaName, TableModel tableModel, Connection conn) throws SQLException {
+
+ // Checking for enum
+ Map<String, String> enumNames = new HashMap<>();
+ for (ColumnModel column : tableModel.getColumns()) {
+ if (column.getEnumValues() != null) {
+ String enumName = getEnumName(schemaName, tableModel.getName(), column.getName());
+ if (!enumTypeExists(enumName)) {
+ createEnumType(enumName, column.getEnumValues());
+ }
+ enumNames.put(column.getName(), enumName);
+ }
+ }
+
StringBuilder querySb = new StringBuilder();
querySb.append("CREATE TABLE IF NOT EXISTS ");
@@ -85,10 +134,14 @@ public class PostgresDBBroker extends DBBrokerTemplate {
querySb.append(cm.getName());
querySb.append(" ");
- String pgsqlType = TypesMapping.getPostgresSQLTypeFromADQLType(cm.getType()).toLowerCase();
- querySb.append(pgsqlType);
- if (pgsqlType.equals("character varying") || pgsqlType.equals("character")) {
- appendSize(querySb, cm.getSize());
+ if (cm.getEnumValues() == null) {
+ String pgsqlType = TypesMapping.getPostgresSQLTypeFromADQLType(cm.getType()).toLowerCase();
+ querySb.append(pgsqlType);
+ if (cm.getSize() != null) {
+ appendSize(querySb, cm.getSize());
+ }
+ } else {
+ querySb.append(enumNames.get(cm.getName()));
}
if (cm.isNullable()) {
@@ -98,6 +151,7 @@ public class PostgresDBBroker extends DBBrokerTemplate {
}
}
}
+ appendCheckConstraints(querySb, tableModel);
querySb.append(")");
@@ -382,4 +436,21 @@ public class PostgresDBBroker extends DBBrokerTemplate {
protected String getAllTablesNamesQuery(String schemaName) {
return String.format("SELECT tablename FROM pg_catalog.pg_tables where schemaname = '%s'", schemaName);
}
+
+ @Override
+ public void alterDataType(String schemaName, String tableName, String columnName, String adqlDataType, Integer size) throws SQLException {
+
+ String psqlDataType = TypesMapping.getPostgresSQLTypeFromADQLType(adqlDataType);
+ if (size != null) {
+ psqlDataType = String.format("%s(%s)", psqlDataType, size);
+ }
+
+ String query = String.format("ALTER TABLE %s.%s ALTER COLUMN %s TYPE %s", escape(schemaName), escape(tableName), escape(columnName), psqlDataType);
+
+ try (Connection connection = dataSource.getConnection();
+ Statement statement = connection.createStatement()) {
+ LOG.debug("Executing query: {}", query);
+ statement.executeUpdate(query);
+ }
+ }
}
diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/model/ColumnModel.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/model/ColumnModel.java
index 904f9cd..670e0f0 100644
--- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/model/ColumnModel.java
+++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/model/ColumnModel.java
@@ -23,9 +23,11 @@
package it.inaf.ia2.tsm.model;
import java.io.Serializable;
+import java.util.List;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlTransient;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
/**
*
@@ -49,6 +51,9 @@ public class ColumnModel implements Serializable {
private String utype;
private String unit;
private boolean principal;
+ private List<String> enumValues;
+ private Integer minValue;
+ private Integer maxValue;
public ColumnModel() {
// default values
@@ -184,6 +189,34 @@ public class ColumnModel implements Serializable {
this.principal = principal;
}
+ @XmlElement(name = "enum-values", nillable = true, required = false)
+ @XmlJavaTypeAdapter(CommaSeparatedListAdapter.class)
+ public List<String> getEnumValues() {
+ return enumValues;
+ }
+
+ public void setEnumValues(List<String> enumValues) {
+ this.enumValues = enumValues;
+ }
+
+ @XmlElement(name = "min", nillable = true, required = false)
+ public Integer getMinValue() {
+ return minValue;
+ }
+
+ public void setMinValue(Integer minValue) {
+ this.minValue = minValue;
+ }
+
+ @XmlElement(name = "max", nillable = true, required = false)
+ public Integer getMaxValue() {
+ return maxValue;
+ }
+
+ public void setMaxValue(Integer maxValue) {
+ this.maxValue = maxValue;
+ }
+
@XmlTransient
public Object getDefaultValue() {
if (defaultValueString == null) {
diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/model/CommaSeparatedListAdapter.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/model/CommaSeparatedListAdapter.java
new file mode 100644
index 0000000..450e734
--- /dev/null
+++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/model/CommaSeparatedListAdapter.java
@@ -0,0 +1,59 @@
+/*
+ * _____________________________________________________________________________
+ *
+ * INAF - OATS National Institute for Astrophysics - Astronomical Observatory of
+ * Trieste INAF - IA2 Italian Center for Astronomical Archives
+ * _____________________________________________________________________________
+ *
+ * Copyright (C) 2017 Istituto Nazionale di Astrofisica
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License Version 3 as published by the
+ * Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package it.inaf.ia2.tsm.model;
+
+import java.util.Arrays;
+import java.util.List;
+import javax.xml.bind.annotation.adapters.XmlAdapter;
+
+/**
+ *
+ * @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
+ */
+public class CommaSeparatedListAdapter extends XmlAdapter<String, List<String>> {
+
+ @Override
+ public List<String> unmarshal(String value) throws Exception {
+ if (value == null) {
+ return null;
+ }
+
+ return Arrays.asList(value.split(","));
+ }
+
+ @Override
+ public String marshal(List<String> values) throws Exception {
+ if (values == null) {
+ return null;
+ }
+ StringBuilder sb = new StringBuilder();
+ boolean first = true;
+ for (String value : values) {
+ if (!first) {
+ sb.append(",");
+ }
+ sb.append(value);
+ }
+ return sb.toString();
+ }
+}
diff --git a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/model/TypesMapping.java b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/model/TypesMapping.java
index ec8988e..484b480 100644
--- a/TASMAN-core/src/main/java/it/inaf/ia2/tsm/model/TypesMapping.java
+++ b/TASMAN-core/src/main/java/it/inaf/ia2/tsm/model/TypesMapping.java
@@ -111,6 +111,26 @@ public class TypesMapping {
}
}
+ public static TypeMapping getTypeMapping(String dataType, DataTypeMode mode) {
+ for (TypeMapping typeMapping : TYPES) {
+ switch (mode) {
+ case ADQL:
+ if (dataType.equals(typeMapping.getAdqlType())) {
+ return typeMapping;
+ }
+ break;
+ case VOTABLE:
+ if (dataType.equals(typeMapping.getVoTableType())) {
+ return typeMapping;
+ }
+ break;
+ default:
+ throw new UnsupportedOperationException("DataTypeMode " + mode + " not supported yet");
+ }
+ }
+ throw new RuntimeException("Unable to get TypeMapping for datatype " + dataType + " [" + mode + "]");
+ }
+
private static abstract class DataTypeFromDBTypeMapper {
private final DataTypeMode mode;
diff --git a/TASMAN-core/src/main/resources/schema_definition/ivoa-1_1.xml b/TASMAN-core/src/main/resources/schema_definition/ivoa-1_1.xml
index 295aa94..8feb50a 100644
--- a/TASMAN-core/src/main/resources/schema_definition/ivoa-1_1.xml
+++ b/TASMAN-core/src/main/resources/schema_definition/ivoa-1_1.xml
@@ -25,7 +25,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
<table name="obscore">
<column name="dataproduct_type">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Logical data product type (image etc.)</description>
<standard>true</standard>
@@ -33,10 +32,10 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
<utype>ObsDataset.dataProductType</utype>
<ucd>meta.id</ucd>
<principal>true</principal>
+ <enum-values>image,cube,spectrum,sed,timeseries,visibility,event,measurements</enum-values>
</column>
<column name="dataproduct_subtype">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Data product specific type</description>
<standard>true</standard>
@@ -55,10 +54,11 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
<utype>ObsDataset.calibLevel</utype>
<ucd>meta.code;obs.calib</ucd>
<principal>true</principal>
+ <min>0</min>
+ <max>4</max>
</column>
<column name="obs_collection">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<nullable>false</nullable>
<description>Name of the data collection</description>
@@ -70,7 +70,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="obs_id">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<nullable>false</nullable>
<description>Internal ID given by the ObsTAP service</description>
@@ -82,7 +81,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="obs_title">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Brief description of dataset in free format</description>
<standard>true</standard>
@@ -103,7 +101,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="obs_creator_name">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Name of the creator of the data</description>
<standard>true</standard>
@@ -114,7 +111,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="obs_creator_did">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>IVOA dataset identifier given by the creator</description>
<standard>true</standard>
@@ -135,7 +131,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="publisher_id">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>IVOA-ID for the Publisher</description>
<standard>true</standard>
@@ -146,7 +141,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="bib_reference">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Service bibliographic reference</description>
<standard>true</standard>
@@ -157,7 +151,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="data_rights">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Public/Secure/Proprietary</description>
<standard>true</standard>
@@ -168,7 +161,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="obs_publisher_did">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<nullable>false</nullable>
<description>Dataset identifier given by the publisher</description>
@@ -180,7 +172,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="access_url">
<type>CLOB</type>
- <size>255</size>
<updatable>true</updatable>
<description>URL used to access (download) dataset</description>
<standard>true</standard>
@@ -191,7 +182,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="access_format">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Content format of the dataset</description>
<standard>true</standard>
@@ -214,7 +204,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="target_name">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Astronomical object observed, if any</description>
<standard>true</standard>
@@ -225,7 +214,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="target_class">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Class of the Target object as in SSA</description>
<standard>true</standard>
@@ -272,7 +260,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="s_region">
<type>REGION</type>
- <size>255</size>
<updatable>true</updatable>
<description>Sky region covered by the data product (expressed in ICRS frame)</description>
<standard>true</standard>
@@ -303,7 +290,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="s_ucd">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>UCD for the nature of the spatial axis (pos or u,v data)</description>
<standard>true</standard>
@@ -314,7 +300,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="s_unit">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Unit used for spatial axis</description>
<standard>true</standard>
@@ -358,7 +343,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="s_calib_status">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Type of calibration along the spatial axis</description>
<standard>true</standard>
@@ -436,7 +420,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="t_calib_status">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Type of time coordinate calibration</description>
<standard>true</standard>
@@ -468,7 +451,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="t_refpos">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Time Axis Reference Position as defined in STC REC</description>
<standard>true</standard>
@@ -560,7 +542,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="em_ucd">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Nature of the spectral axis</description>
<standard>true</standard>
@@ -571,7 +552,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="em_unit">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Units along the spectral axis</description>
<standard>true</standard>
@@ -582,7 +562,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="em_calib_status">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Type of spectral coord calibration</description>
<standard>true</standard>
@@ -593,7 +572,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="o_ucd">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>UCD of the observable axis (e.g. phot.flux.density, phot.count, etc.)</description>
<standard>true</standard>
@@ -604,7 +582,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="o_unit">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Units used for the observable values</description>
<standard>true</standard>
@@ -615,7 +592,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="o_calib_status">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Type of calibration for the observable coordinate</description>
<standard>true</standard>
@@ -637,7 +613,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="pol_states">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>List of polarization states present in the data file or NULL if not applicable</description>
<standard>true</standard>
@@ -658,7 +633,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="facility_name">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Name of the facility used for this observation</description>
<standard>true</standard>
@@ -669,7 +643,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="instrument_name">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>The name of the instrument used for the observation</description>
<standard>true</standard>
@@ -680,7 +653,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="proposal_id">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>Identifier of proposal to which observation belongs</description>
<standard>false</standard>
diff --git a/TASMAN-core/src/main/resources/schema_definition/tap_schema-1.xml b/TASMAN-core/src/main/resources/schema_definition/tap_schema-1.xml
index 07dc153..85d28d9 100644
--- a/TASMAN-core/src/main/resources/schema_definition/tap_schema-1.xml
+++ b/TASMAN-core/src/main/resources/schema_definition/tap_schema-1.xml
@@ -34,7 +34,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="utype">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>lists the utypes of schemas in the tableset</description>
<standard>true</standard>
@@ -75,7 +74,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="utype">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>lists the utype of tables in the tableset</description>
<standard>true</standard>
@@ -131,7 +129,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="utype">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>lists the utypes of columns in the tableset</description>
<standard>true</standard>
@@ -208,7 +205,6 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
</column>
<column name="utype">
<type>VARCHAR</type>
- <size>255</size>
<updatable>true</updatable>
<description>lists the utype of keys in the tableset</description>
<standard>true</standard>
diff --git a/TASMAN-core/src/main/resources/schema_definition/tap_schema-1_1.xml b/TASMAN-core/src/main/resources/schema_definition/tap_schema-1_1.xml
index f5b8a73..ca0050a 100644
--- a/TASMAN-core/src/main/resources/schema_definition/tap_schema-1_1.xml
+++ b/TASMAN-core/src/main/resources/schema_definition/tap_schema-1_1.xml
@@ -34,13 +34,11 @@ Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
<type>VARCHAR</type>
<updatable>false</updatable>
<key>arraysize</key>
- <size>255</size>
<standard>true</standard>
</column>
<column name="xtype">
<type>VARCHAR</type>
<updatable>true</updatable>
- <size>255</size>
<standard>true</standard>
</column>
<column name="column_index">
diff --git a/TASMAN-core/src/test/java/it/inaf/ia2/tsm/TestAll.java b/TASMAN-core/src/test/java/it/inaf/ia2/tsm/TestAll.java
index ee2b2b2..3cafafd 100644
--- a/TASMAN-core/src/test/java/it/inaf/ia2/tsm/TestAll.java
+++ b/TASMAN-core/src/test/java/it/inaf/ia2/tsm/TestAll.java
@@ -459,12 +459,6 @@ public class TestAll {
operations = new UpdateOperations(tapSchema);
assertFalse(operations.getHasOperations());
- // Checking size and arraysize detection
- Column descriptionColumn = tapSchema.getChild(tapSchema.getName()).getChild("columns").getChild("utype");
- assertEquals("char", descriptionColumn.getProperty("datatype").getValue(String.class));
- assertEquals(255, descriptionColumn.getProperty("size").getValue(Integer.class));
- assertEquals("255*", descriptionColumn.getProperty("arraysize").getValue(String.class));
-
// reloading
LOG.debug("----- Reloading saved TAP_SCHEMA -----");
tapSchema = new TapSchema(dbWrapper, settings, true);
diff --git a/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/ConsistencyChecksBean.java b/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/ConsistencyChecksBean.java
index ec52620..47f7616 100644
--- a/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/ConsistencyChecksBean.java
+++ b/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/ConsistencyChecksBean.java
@@ -53,14 +53,14 @@ public class ConsistencyChecksBean implements Serializable {
this.tapSchema = tapSchema;
}
- public boolean isTapSchemaContainsOnlyTapSchema() {
- return tapSchema.getChildren().size() == 1
- && tapSchema.getChildren().get(0).getName().equals(tapSchema.getName());
- }
-
public String proceed() throws SQLException {
TapSchemaMender.amendTapSchema(tapSchema);
tapSchema.save();
+ tapSchema.load();
+ if (tapSchema.getConsistencyChecks().isInconsistent()) {
+ // Show inconsistencies again
+ return null;
+ }
tapSchemaEditingBean.setTapSchema(tapSchema);
return "tapSchemaEditing.xhtml?faces-redirect=true";
}
diff --git a/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/TapSchemaEditingBean.java b/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/TapSchemaEditingBean.java
index e4cbfee..e404102 100644
--- a/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/TapSchemaEditingBean.java
+++ b/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/TapSchemaEditingBean.java
@@ -404,6 +404,18 @@ public class TapSchemaEditingBean implements Serializable {
}
}
+ public boolean isSchemaRemovable(Schema schema) {
+ return !schema.getName().equals(tapSchema.getName()) && !schema.getName().equals("ivoa");
+ }
+
+ public boolean isColumnRemovable(Column column) {
+ boolean parentSchemaRemovable = isSchemaRemovable(column.getParent().getParent());
+ if (!parentSchemaRemovable) {
+ return !column.isMandatory();
+ }
+ return true;
+ }
+
public void reload() {
if (tapSchema.exists()) {
diff --git a/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/TapSchemaLoader.java b/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/TapSchemaLoader.java
index 0c3559d..9461743 100644
--- a/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/TapSchemaLoader.java
+++ b/TASMAN-webapp/src/main/java/it/inaf/ia2/tsm/webapp/TapSchemaLoader.java
@@ -22,6 +22,7 @@
*/
package it.inaf.ia2.tsm.webapp;
+import it.inaf.ia2.tsm.ConsistencyChecks;
import it.inaf.ia2.tsm.TapSchema;
import it.inaf.ia2.tsm.TapSchemaSettings;
import it.inaf.ia2.tsm.datalayer.DBBroker;
@@ -84,14 +85,22 @@ public class TapSchemaLoader implements Serializable {
DataTypeMode dataTypeMode = SchemaModels.getTapSchemaModel(tapCredentials.getTapSchemaVersion()).getDataTypeMode();
DBBroker broker = DBBrokerFactory.getDBBroker(dbWrapper.getTapSchemaDataSourceWrapper(), dataTypeMode);
boolean tapSchemaExists = false;
+ boolean ivoaSchemaExists = false;
for (String schemaName : broker.getAllSchemaNames()) {
if (schemaName.equals(tapCredentials.getTapSchemaName())) {
tapSchemaExists = true;
+ }
+ // Check for ivoa schema existence only if the user wants obscore
+ if (tapCredentials.isHasObscore() && schemaName.equals("ivoa")) {
+ ivoaSchemaExists = true;
+ }
+ if (tapSchemaExists && ivoaSchemaExists) {
break;
}
}
CustomPartialResponseWriter.getCurrentInstance().addCustomJSUpdate("tap_schema_existence", String.valueOf(tapSchemaExists));
+ CustomPartialResponseWriter.getCurrentInstance().addCustomJSUpdate("ivoa_schema_existence", String.valueOf(ivoaSchemaExists));
if (tapSchemaExists) {
edit();
@@ -174,7 +183,8 @@ public class TapSchemaLoader implements Serializable {
}
public String openLoaded() {
- if (loadedTapSchema.getConsistencyChecks().isInconsistent()) {
+ ConsistencyChecks checks = loadedTapSchema.getConsistencyChecks();
+ if (checks != null && (checks.isInconsistent() || checks.isHasWarnings())) {
consistencyChecksBean.setTapSchema(loadedTapSchema);
return "consistencyChecks.xhtml?faces-redirect=true";
} else {
diff --git a/TASMAN-webapp/src/main/webapp/consistencyChecks.xhtml b/TASMAN-webapp/src/main/webapp/consistencyChecks.xhtml
index 80e28b2..36dbd1c 100644
--- a/TASMAN-webapp/src/main/webapp/consistencyChecks.xhtml
+++ b/TASMAN-webapp/src/main/webapp/consistencyChecks.xhtml
@@ -11,7 +11,13 @@
<div class="container">
<div class="row">
<div class="col-xs-12">
- <h1>Consistency problems detected in #{consistency.tapSchema.name}</h1>
+ <h:panelGroup rendered="#{consistency.tapSchema.consistencyChecks.inconsistent}">
+ <h1>Consistency problems detected in #{consistency.tapSchema.name}</h1>
+ </h:panelGroup>
+ <h:panelGroup rendered="#{!consistency.tapSchema.consistencyChecks.inconsistent and consistency.tapSchema.consistencyChecks.hasWarnings}">
+ <h1>Warnings - #{consistency.tapSchema.name}</h1>
+ </h:panelGroup>
+
<h:form id="main">
<h:panelGroup rendered="#{consistency.tapSchema.consistencyChecks.inconsistencies.size() gt 0}">
@@ -116,21 +122,51 @@
</ul>
</h:panelGroup>
+ <h:panelGroup rendered="#{consistency.tapSchema.consistencyChecks.wrongDataTypes.size() gt 0}">
+ <h2>Wrong data type in original table</h2>
+ <table class="table table-bordered">
+ <thead>
+ <tr>
+ <th>Schema</th>
+ <th>Table</th>
+ <th>Column</th>
+ <th>Wrong data type</th>
+ <th>Correct data type</th>
+ </tr>
+ </thead>
+ <tbody>
+ <ui:repeat value="#{consistency.tapSchema.consistencyChecks.wrongDataTypes}" var="wdt">
+ <tr>
+ <td>${wdt.columnHolder.schemaName}</td>
+ <td>${wdt.columnHolder.tableName}</td>
+ <td>${wdt.columnHolder.columnName}</td>
+ <td class="text-danger">${wdt.wrongDataType}</td>
+ <td class="text-success">${wdt.correctDataType}</td>
+ </tr>
+ </ui:repeat>
+ </tbody>
+ </table>
+ </h:panelGroup>
+
<h:panelGroup rendered="#{consistency.tapSchema.consistencyChecks.missingObscore or consistency.tapSchema.consistencyChecks.obscoreToAdd}">
<h2>ObsCore to add</h2>
<span class="glyphicon glyphicon-check"></span>
</h:panelGroup>
<br/>
- <h:panelGroup rendered="#{consistency.tapSchemaContainsOnlyTapSchema}" layout="block" class="alert alert-danger text-center">
- <span class="glyphicon glyphicon-warning-sign"></span>
- <strong>If you proceed this TAP_SCHEMA will contain only itself! You may have selected wrong source credentials.</strong>
+ <h:panelGroup rendered="#{consistency.tapSchema.consistencyChecks.unaddedOptionalColumns.size() gt 0}">
+ <h2>Unadded optional columns</h2>
+ <ul>
+ <ui:repeat value="#{consistency.tapSchema.consistencyChecks.unaddedOptionalColumns.toArray()}" var="column">
+ <li>${column}</li>
+ </ui:repeat>
+ </ul>
</h:panelGroup>
- <div class="alert alert-warning text-center">
+ <h:panelGroup rendered="#{consistency.tapSchema.consistencyChecks.inconsistent}" layout="block" class="alert alert-warning text-center">
<span class="glyphicon glyphicon-warning-sign"></span>
<strong>If you proceed the TAP_SCHEMA Manager will fix these values.</strong>
- </div>
+ </h:panelGroup>
<div class="text-center">
<h:commandLink action="#{consistency.back()}" class="btn btn-default">
diff --git a/TASMAN-webapp/src/main/webapp/credentialsEditing.xhtml b/TASMAN-webapp/src/main/webapp/credentialsEditing.xhtml
index 577bccd..f20e2ae 100644
--- a/TASMAN-webapp/src/main/webapp/credentialsEditing.xhtml
+++ b/TASMAN-webapp/src/main/webapp/credentialsEditing.xhtml
@@ -300,7 +300,11 @@
<h4 class="modal-title">Create TAP_SCHEMA</h4>
</div>
<div class="modal-body">
- The TAP_SCHEMA <strong>#{tapSchemaLoader.tapCredentials.tapSchemaName}</strong> doesn't exists. Do you want to create it?
+ <p>The TAP_SCHEMA <strong>#{tapSchemaLoader.tapCredentials.tapSchemaName}</strong> doesn't exists. Do you want to create it?</p>
+ <p class="ivoa-schema-warning text-warning hide">
+ <span class="glyphicon glyphicon-alert"></span>
+ <strong>WARNING</strong>: existing ivoa schema detected. Next changes could alter an existing ObsCore table.
+ </p>
</div>
<div class="modal-footer">
<h:commandButton class="btn btn-success" value="Yes" action="#{tapSchemaLoader.create()}">
diff --git a/TASMAN-webapp/src/main/webapp/resources/js/async-loader.js b/TASMAN-webapp/src/main/webapp/resources/js/async-loader.js
index 25697d8..e91652e 100644
--- a/TASMAN-webapp/src/main/webapp/resources/js/async-loader.js
+++ b/TASMAN-webapp/src/main/webapp/resources/js/async-loader.js
@@ -34,6 +34,9 @@
if ($tsExistenceEl.text() === 'true') {
periodicCheck();
} else {
+ var $ivoaExistenceEl = $(event.responseXML).find('jsupdate[src="ivoa_schema_existence"]');
+ var ivoaSchemaExists = $ivoaExistenceEl.text() === 'true';
+ $('#modal-confirm-ts-creation .ivoa-schema-warning').toggleClass('hide', !ivoaSchemaExists)
$('#modal-confirm-ts-creation').modal('show');
}
}
diff --git a/TASMAN-webapp/src/main/webapp/tapSchemaEditing.xhtml b/TASMAN-webapp/src/main/webapp/tapSchemaEditing.xhtml
index 387d45d..9de0d52 100644
--- a/TASMAN-webapp/src/main/webapp/tapSchemaEditing.xhtml
+++ b/TASMAN-webapp/src/main/webapp/tapSchemaEditing.xhtml
@@ -77,11 +77,11 @@
<ul class="nav nav-tabs" id="schemas" role="tablist">
<ui:repeat value="#{tapSchemaEditing.tapSchema.addedOrRemovedChildren}" var="schema">
- <li role="presentation" class="removable-tab #{schema.name eq tapSchemaEditing.tapSchema.name ? 'unremovable-tab': ''} #{tapSchemaEditing.selectedSchema.name eq schema.name ? 'active': ''} #{tapSchemaEditing.toRemove(schema) ? 'to-remove' : ''}">
+ <li role="presentation" class="removable-tab #{tapSchemaEditing.isSchemaRemovable(schema) ? '': 'unremovable-tab'} #{tapSchemaEditing.selectedSchema.name eq schema.name ? 'active': ''} #{tapSchemaEditing.toRemove(schema) ? 'to-remove' : ''}">
<h:commandLink role="tab" action="#{tapSchemaEditing.setSelectedSchema(schema)}">
<span class="#{tapSchemaEditing.toRemove(schema) ? 'strikeout':''}">#{schema.name}</span>
<f:ajax execute="@form" render="@form" />
- <h:commandButton class="remove-btn" rendered="#{!tapSchemaEditing.toRemove(schema) and schema.name ne tapSchemaEditing.tapSchema.name}" action="#{tapSchemaEditing.tapSchema.removeChild(schema.name)}" value="×" onclick="TSM.stopPropagation(event)">
+ <h:commandButton class="remove-btn" rendered="#{!tapSchemaEditing.toRemove(schema) and tapSchemaEditing.isSchemaRemovable(schema)}" action="#{tapSchemaEditing.tapSchema.removeChild(schema.name)}" value="×" onclick="TSM.stopPropagation(event)">
<f:ajax execute="@form" render="@form" />
</h:commandButton>
</h:commandLink>
@@ -144,11 +144,11 @@
<h:panelGroup id="tables_wrapper">
<ul class="nav nav-tabs" id="tables" role="tablist">
<ui:repeat value="#{tapSchemaEditing.selectedSchema.addedOrRemovedChildren}" var="table">
- <li role="presentation" class="removable-tab #{table.parent.name eq tapSchemaEditing.tapSchema.name ? 'unremovable-tab': ''} #{tapSchemaEditing.selectedTable.name eq table.name ? 'active':''} #{tapSchemaEditing.toRemove(table) ? 'to-remove':''}">
+ <li role="presentation" class="removable-tab #{tapSchemaEditing.isSchemaRemovable(table.parent) ? '': 'unremovable-tab'} #{tapSchemaEditing.selectedTable.name eq table.name ? 'active':''} #{tapSchemaEditing.toRemove(table) ? 'to-remove':''}">
<h:commandLink role="tab" action="#{tapSchemaEditing.setSelectedTable(table)}">
<span class="#{tapSchemaEditing.toRemove(table) ? 'strikeout':''}">#{table.name}</span>
<f:ajax execute="@form" render=":main:tables_wrapper" />
- <h:commandButton class="remove-btn" rendered="#{!tapSchemaEditing.toRemove(table) and table.parent.name ne tapSchemaEditing.tapSchema.name}" action="#{tapSchemaEditing.selectedSchema.removeChild(table.name)}" value="×" onclick="TSM.stopPropagation(event)">
+ <h:commandButton class="remove-btn" rendered="#{!tapSchemaEditing.toRemove(table) and tapSchemaEditing.isSchemaRemovable(table.parent)}" action="#{tapSchemaEditing.selectedSchema.removeChild(table.name)}" value="×" onclick="TSM.stopPropagation(event)">
<f:ajax execute="@form" render=":main:tables_wrapper" />
</h:commandButton>
</h:commandLink>
@@ -236,7 +236,7 @@
<ui:repeat value="#{tapSchemaEditing.selectedTable.addedOrRemovedChildren}" var="column" id="columns-list">
<li role="presentation" class="#{tapSchemaEditing.selectedColumn.name eq column.name ? 'active': ''}">
<h:commandLink role="tab" action="#{tapSchemaEditing.setSelectedColumn(column)}" id="column-selector">
- <h:commandButton class="btn btn-link remove-btn" disabled="#{tapSchemaEditing.toRemove(column) or column.parent.parent.name eq tapSchemaEditing.tapSchema.name}" value="×" onclick="TSM.stopPropagation(event)" id="column-remover">
+ <h:commandButton class="btn btn-link remove-btn" disabled="#{!tapSchemaEditing.isColumnRemovable(column)}" value="×" onclick="TSM.stopPropagation(event)" id="column-remover">
<f:ajax execute="@form" render=":main:tables_wrapper" listener="#{tapSchemaEditing.removeColumn(column.name)}" />
</h:commandButton>
--
GitLab