From e6937fa6cc060b3012ebe5e6b19341400f91aad7 Mon Sep 17 00:00:00 2001 From: acpaquette Date: Tue, 8 Oct 2024 17:16:59 -0700 Subject: [PATCH 1/5] Add ability to pass column types to csv2table --- isis/src/base/apps/csv2table/csv2table.xml | 15 ++++ isis/src/base/apps/csv2table/main.cpp | 91 ++++++++++++++++++++-- 2 files changed, 101 insertions(+), 5 deletions(-) diff --git a/isis/src/base/apps/csv2table/csv2table.xml b/isis/src/base/apps/csv2table/csv2table.xml index 10624eee4d..faea6ae54b 100644 --- a/isis/src/base/apps/csv2table/csv2table.xml +++ b/isis/src/base/apps/csv2table/csv2table.xml @@ -39,6 +39,9 @@ Added ability to convert CSV files with indicies into table field arrays instead of individual table fields. + + Added the ability to pass types for each column in the CSV + @@ -94,6 +97,18 @@ the cube it will be overwritten. + + + string + Column types of the table + + ISIS command line list of Tablefield types. Specified as such, '(type1, type2)' + where the allowed types are "Double", "Integer", "Float", "Text". + + + + + diff --git a/isis/src/base/apps/csv2table/main.cpp b/isis/src/base/apps/csv2table/main.cpp index 2ba1de8cf5..99239fa969 100644 --- a/isis/src/base/apps/csv2table/main.cpp +++ b/isis/src/base/apps/csv2table/main.cpp @@ -60,6 +60,49 @@ void IsisMain() { "] rows and [" + toString(numColumns) +"] columns."; throw IException(IException::User, msg, _FILEINFO_); } + + std::vector fieldTypes; + ui.GetAsString("coltypes", fieldTypes); + + std::vector tableTypes; + if (fieldTypes.size() == 1 && fieldTypes[0] == "") { + for (int i = 0; i < numColumns; i++) { + tableTypes.push_back(TableField::Double); + } + } + else { + if (fieldTypes.size() == numColumns) { + for (QString type: fieldTypes) { + QString upper_type = type.toUpper(); + if (upper_type == "INTEGER") { + tableTypes.push_back(TableField::Type::Integer); + } + else if (upper_type == "DOUBLE") { + tableTypes.push_back(TableField::Type::Double); + } + else if (upper_type == "TEXT") { + tableTypes.push_back(TableField::Type::Text); + } + else if (upper_type == "REAL") { + tableTypes.push_back(TableField::Type::Real); + } + else { + QString msg = "Field [" + type + "] cannot be translated. Accepted types are " + "Integer, Double, Text, and Real"; + throw IException(IException::User, msg, _FILEINFO_); + } + } + } + else { + int numFields = fieldTypes.size(); + QString msg = "Number of fields provided does not equal the number of columns in the CSV. " + "Number of fields [" + toString(numFields) + + "] vs Number of Columns [" + toString(numColumns) + "]"; + throw IException(IException::User, msg, _FILEINFO_); + } + } + + CSVReader::CSVAxis header = reader.getHeader(); // Construct an empty table with the CSV header as field names @@ -76,7 +119,7 @@ void IsisMain() { // If the next column header is different, create a field for this one QRegularExpressionMatch nextMatch = (columnIndex0)?(index.toInt()+1):1); + TableField columnField(name, tableTypes[columnIndex], (index.length()>0)?(index.toInt()+1):1); tableRow += columnField; } } @@ -90,14 +133,52 @@ void IsisMain() { CSVReader::CSVAxis csvRow = reader.getRow(rowIndex); for (int columnIndex = 0, fieldIndex = 0; columnIndex < numColumns; ) { if (tableRow[fieldIndex].size() == 1) { - tableRow[fieldIndex] = toDouble(csvRow[columnIndex++]); + switch(tableTypes[columnIndex]) { + case TableField::Type::Integer: + tableRow[fieldIndex] = toInt(csvRow[columnIndex++]); + break; + case TableField::Type::Double: + tableRow[fieldIndex] = toDouble(csvRow[columnIndex++]); + break; + case TableField::Type::Text: + tableRow[fieldIndex] = csvRow[columnIndex++]; + break; + case TableField::Type::Real: + tableRow[fieldIndex] = toDouble(csvRow[columnIndex++]); + break; + } } else { + std::vector intVector; std::vector dblVector; - for (int arrayLen = 0; arrayLen < tableRow[fieldIndex].size(); arrayLen++) { - dblVector.push_back(toDouble(csvRow[columnIndex++])); + std::vector realVector; + QString strMsg = "TableRecord can't handle list of Strings"; + switch(tableTypes[columnIndex]) { + case TableField::Type::Integer: + for (int arrayLen = 0; arrayLen < tableRow[fieldIndex].size(); arrayLen++) { + intVector.push_back(toInt(csvRow[columnIndex++])); + } + tableRow[fieldIndex] = intVector; + break; + case TableField::Type::Double: + for (int arrayLen = 0; arrayLen < tableRow[fieldIndex].size(); arrayLen++) { + dblVector.push_back(toDouble(csvRow[columnIndex++])); + } + tableRow[fieldIndex] = dblVector; + break; + case TableField::Type::Text: + throw IException(IException::User, strMsg, _FILEINFO_); + break; + case TableField::Type::Real: + for (int arrayLen = 0; arrayLen < tableRow[fieldIndex].size(); arrayLen++) { + realVector.push_back(toDouble(csvRow[columnIndex++])); + } + tableRow[fieldIndex] = realVector; + break; } - tableRow[fieldIndex] = dblVector; + intVector.clear(); + dblVector.clear(); + realVector.clear(); } fieldIndex++; } From ccb8e530254be89ecddc0082cc20b3ae620694d1 Mon Sep 17 00:00:00 2001 From: acpaquette Date: Sun, 8 Dec 2024 11:35:19 -0700 Subject: [PATCH 2/5] Added gtests for csv2table --- isis/src/base/apps/csv2table/main.cpp | 228 +------------ isis/tests/FunctionalTestsCsv2Table.cpp | 305 ++++++++++++++++++ isis/tests/data/csv2table/empty.csv | 0 isis/tests/data/csv2table/test.csv | 4 + isis/tests/data/csv2table/test_2.csv | 4 + isis/tests/data/csv2table/test_arrays.csv | 4 + .../tests/data/csv2table/test_type_arrays.csv | 4 + 7 files changed, 333 insertions(+), 216 deletions(-) create mode 100644 isis/tests/FunctionalTestsCsv2Table.cpp create mode 100644 isis/tests/data/csv2table/empty.csv create mode 100644 isis/tests/data/csv2table/test.csv create mode 100644 isis/tests/data/csv2table/test_2.csv create mode 100644 isis/tests/data/csv2table/test_arrays.csv create mode 100644 isis/tests/data/csv2table/test_type_arrays.csv diff --git a/isis/src/base/apps/csv2table/main.cpp b/isis/src/base/apps/csv2table/main.cpp index 99239fa969..b0a65f9bfb 100644 --- a/isis/src/base/apps/csv2table/main.cpp +++ b/isis/src/base/apps/csv2table/main.cpp @@ -1,227 +1,23 @@ -/** - * @file - * $Revision: 1.8 $ - * $Date: 2010/04/08 15:28:20 $ - * - * Unless noted otherwise, the portions of Isis written by the USGS are public - * domain. See individual third-party library and package descriptions for - * intellectual property information,user agreements, and related information. - * - * Although Isis has been used by the USGS, no warranty, expressed or implied, - * is made by the USGS as to the accuracy and functioning of such software - * and related material nor shall the fact of distribution constitute any such - * warranty, and no responsibility is assumed by the USGS in connection - * therewith. - * - * For additional information, launch - * $ISISROOT/doc//documents/Disclaimers/Disclaimers.html in a browser or see - * the Privacy & Disclaimers page on the Isis website, - * http://isis.astrogeology.usgs.gov, and the USGS privacy and disclaimers on - * http://www.usgs.gov/privacy.html. - */ +/** This is free and unencumbered software released into the public domain. -#include "Isis.h" +The authors of ISIS do not claim copyright on the contents of this file. +For more details about the LICENSE terms and the AUTHORS, you will +find files of those names at the top level of this repository. **/ -#include +/* SPDX-License-Identifier: CC0-1.0 */ +#include "Isis.h" -#include -#include +#include "csv2table.h" -#include "Cube.h" -#include "CSVReader.h" +#include "Application.h" #include "IException.h" -#include "IString.h" #include "Pvl.h" -#include "PvlObject.h" -#include "Table.h" -#include "TableField.h" -#include "TableRecord.h" +using namespace std; using namespace Isis; - void IsisMain() { UserInterface &ui = Application::GetUserInterface(); - - // Read the CSV file and get the header - QString csvFileName = ui.GetFileName("csv"); - CSVReader reader; - try { - reader = CSVReader(csvFileName, true); - } - catch(IException &e) { - QString msg = "Failed to read CSV file [" + csvFileName + "]."; - throw IException(e, IException::Io, msg, _FILEINFO_); - } - int numColumns = reader.columns(); - int numRows = reader.rows(); - if (numColumns < 1 || numRows < 1) { - QString msg = "CSV file does not have data.\nFile has [" + toString(numRows) + - "] rows and [" + toString(numColumns) +"] columns."; - throw IException(IException::User, msg, _FILEINFO_); - } - - std::vector fieldTypes; - ui.GetAsString("coltypes", fieldTypes); - - std::vector tableTypes; - if (fieldTypes.size() == 1 && fieldTypes[0] == "") { - for (int i = 0; i < numColumns; i++) { - tableTypes.push_back(TableField::Double); - } - } - else { - if (fieldTypes.size() == numColumns) { - for (QString type: fieldTypes) { - QString upper_type = type.toUpper(); - if (upper_type == "INTEGER") { - tableTypes.push_back(TableField::Type::Integer); - } - else if (upper_type == "DOUBLE") { - tableTypes.push_back(TableField::Type::Double); - } - else if (upper_type == "TEXT") { - tableTypes.push_back(TableField::Type::Text); - } - else if (upper_type == "REAL") { - tableTypes.push_back(TableField::Type::Real); - } - else { - QString msg = "Field [" + type + "] cannot be translated. Accepted types are " - "Integer, Double, Text, and Real"; - throw IException(IException::User, msg, _FILEINFO_); - } - } - } - else { - int numFields = fieldTypes.size(); - QString msg = "Number of fields provided does not equal the number of columns in the CSV. " - "Number of fields [" + toString(numFields) + - "] vs Number of Columns [" + toString(numColumns) + "]"; - throw IException(IException::User, msg, _FILEINFO_); - } - } - - - CSVReader::CSVAxis header = reader.getHeader(); - - // Construct an empty table with the CSV header as field names - // Collect identical field names together, including those with (###) at the end, so a single - // table field with multiple values can be created. - TableRecord tableRow; - QRegularExpression rex(R"((?\w+)(\((?[0-9]*)\)|))"); - for (int columnIndex = 0; columnIndex < numColumns; columnIndex++) { - QRegularExpressionMatch match = rex.match(header[columnIndex]); - if (match.hasMatch()) { - QString name = match.captured("name"); - QString index = match.captured("index"); - - // If the next column header is different, create a field for this one - QRegularExpressionMatch nextMatch = (columnIndex0)?(index.toInt()+1):1); - tableRow += columnField; - } - } - } - - QString tableName = ui.GetString("tablename"); - Table table(tableName, tableRow); - - // Fill the table from the csv - for (int rowIndex = 0; rowIndex < numRows; rowIndex++) { - CSVReader::CSVAxis csvRow = reader.getRow(rowIndex); - for (int columnIndex = 0, fieldIndex = 0; columnIndex < numColumns; ) { - if (tableRow[fieldIndex].size() == 1) { - switch(tableTypes[columnIndex]) { - case TableField::Type::Integer: - tableRow[fieldIndex] = toInt(csvRow[columnIndex++]); - break; - case TableField::Type::Double: - tableRow[fieldIndex] = toDouble(csvRow[columnIndex++]); - break; - case TableField::Type::Text: - tableRow[fieldIndex] = csvRow[columnIndex++]; - break; - case TableField::Type::Real: - tableRow[fieldIndex] = toDouble(csvRow[columnIndex++]); - break; - } - } - else { - std::vector intVector; - std::vector dblVector; - std::vector realVector; - QString strMsg = "TableRecord can't handle list of Strings"; - switch(tableTypes[columnIndex]) { - case TableField::Type::Integer: - for (int arrayLen = 0; arrayLen < tableRow[fieldIndex].size(); arrayLen++) { - intVector.push_back(toInt(csvRow[columnIndex++])); - } - tableRow[fieldIndex] = intVector; - break; - case TableField::Type::Double: - for (int arrayLen = 0; arrayLen < tableRow[fieldIndex].size(); arrayLen++) { - dblVector.push_back(toDouble(csvRow[columnIndex++])); - } - tableRow[fieldIndex] = dblVector; - break; - case TableField::Type::Text: - throw IException(IException::User, strMsg, _FILEINFO_); - break; - case TableField::Type::Real: - for (int arrayLen = 0; arrayLen < tableRow[fieldIndex].size(); arrayLen++) { - realVector.push_back(toDouble(csvRow[columnIndex++])); - } - tableRow[fieldIndex] = realVector; - break; - } - intVector.clear(); - dblVector.clear(); - realVector.clear(); - } - fieldIndex++; - } - table += tableRow; - } - - // If a set of additional label keywords was given then add them to the table's pvl description - if (ui.WasEntered("label")) { - QString labelPvlFilename = ui.GetFileName("label"); - Pvl labelPvl; - try { - labelPvl.read(labelPvlFilename); - } - catch(IException &e) { - QString msg = "Failed to read PVL label file [" + labelPvlFilename + "]."; - throw IException(e, IException::Io, msg, _FILEINFO_); - } - - PvlObject &tableLabel = table.Label(); - for (int keyIndex = 0; keyIndex < labelPvl.keywords(); keyIndex++) { - tableLabel.addKeyword(labelPvl[keyIndex]); - } - } - - // Write the table to the cube - QString outCubeFileName(ui.GetCubeName("to")); - Cube outCube; - try { - outCube.open(outCubeFileName, "rw"); - } - catch(IException &e) { - QString msg = "Could not open output cube [" + outCubeFileName + "]."; - throw IException(e, IException::Io, msg, _FILEINFO_); - } - - try { - outCube.write(table); - } - catch(IException &e) { - QString msg = "Could not write output table [" + tableName + - "] to output cube [" + outCubeFileName + "]."; - throw IException(e, IException::Io, msg, _FILEINFO_); - } - - outCube.close(); -} + Pvl appLog; + csv2table(ui, &appLog); +} \ No newline at end of file diff --git a/isis/tests/FunctionalTestsCsv2Table.cpp b/isis/tests/FunctionalTestsCsv2Table.cpp new file mode 100644 index 0000000000..29f50b8199 --- /dev/null +++ b/isis/tests/FunctionalTestsCsv2Table.cpp @@ -0,0 +1,305 @@ +#include "CameraFixtures.h" +#include "CubeFixtures.h" +#include "Pvl.h" +#include "PvlGroup.h" +#include "TestUtilities.h" +#include "Table.h" +#include "TableRecord.h" + +#include "csv2table.h" + +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +using namespace Isis; + +static QString APP_XML = FileName("$ISISROOT/bin/xml/csv2table.xml").expanded(); + +TEST_F(DefaultCube, FunctionalTestCsv2TableLabel) { + QTemporaryDir tempDir; + QString csvfile = "data/csv2table/test.csv"; + QString cubePath = testCube->fileName(); + testCube->close(); + QVector args = {"label="+cubePath, "to="+cubePath, + "csv="+csvfile, "tablename=TestTable"}; + + UserInterface options(APP_XML, args); + try { + csv2table(options); + } + catch (IException &e) { + FAIL() << "Unable to open image: " << e.what() << std::endl; + } + + testCube->open(cubePath); + + Table testTable("Temp"); + try { + testTable = testCube->readTable("TestTable"); + } catch(IException &e) { + std::string msg = "Failed to find/read TestTable"; + throw IException(e, IException::Unknown, msg, _FILEINFO_); + } + EXPECT_EQ(testTable.Records(), 3); + EXPECT_EQ(testTable.RecordFields(), 6); + EXPECT_EQ(testTable.RecordSize(), 48); + std::vector expected_values = {10, 0, 11, 0.2, 0.4, -5, + .45, 0.45, -12.58746324, 2, 7, -10, + 3, -1000000, 1000000, 100, 0.45678, 11}; + + for (int i = 0; i < testTable.Records(); i++) { + TableRecord record = testTable[i]; + for (int j = 0; j < record.Fields(); j++) { + EXPECT_EQ(expected_values[(i * record.Fields()) + j], double(record[j])); + } + } +} + +TEST_F(DefaultCube, FunctionalTestCsv2TableArrays) { + QTemporaryDir tempDir; + QString csvfile = "data/csv2table/test_arrays.csv"; + QString cubePath = testCube->fileName(); + testCube->close(); + QVector args = {"label="+cubePath, "to="+cubePath, + "csv="+csvfile, "tablename=TestTableArrays"}; + + UserInterface options(APP_XML, args); + try { + csv2table(options); + } + catch (IException &e) { + FAIL() << "Unable to open image: " << e.what() << std::endl; + } + + // Cube oCube(testCube->fileName(), "r"); + testCube->open(cubePath); + + Table testTable("Temp"); + try { + testTable = testCube->readTable("TestTableArrays"); + } catch(IException &e) { + std::string msg = "Failed to find/read TestTableArrays"; + throw IException(e, IException::Unknown, msg, _FILEINFO_); + } + EXPECT_EQ(testTable.Records(), 3); + EXPECT_EQ(testTable.RecordFields(), 4); + EXPECT_EQ(testTable.RecordSize(), 48); + + // Check array components + std::vector truth = {0.0, 11.0}; + EXPECT_TRUE(std::vector(testTable[0][1]) == truth); + truth = {0.45, -12.58746324}; + EXPECT_TRUE(std::vector(testTable[1][1]) == truth); + truth = {-1000000, 1000000}; + EXPECT_TRUE(std::vector(testTable[2][1]) == truth); + truth = {0.4,-5}; + EXPECT_TRUE(std::vector(testTable[0][3]) == truth); + truth = {7,-10}; + EXPECT_TRUE(std::vector(testTable[1][3]) == truth); + truth = {0.45678,11}; + EXPECT_TRUE(std::vector(testTable[2][3]) == truth); + + // Check non-array components + EXPECT_EQ(double(testTable[0][0]), 10.0); + EXPECT_EQ(double(testTable[1][0]), 0.45); + EXPECT_EQ(double(testTable[2][0]), 3.0); + EXPECT_EQ(double(testTable[0][2]), 0.2); + EXPECT_EQ(double(testTable[1][2]), 2.0); + EXPECT_EQ(double(testTable[2][2]), 100.0); +} + +TEST_F(DefaultCube, FunctionalTestCsv2TableOverwrite) { + QTemporaryDir tempDir; + QString csvfile = "data/csv2table/test.csv"; + QString cubePath = testCube->fileName(); + testCube->close(); + QVector args = {"label="+cubePath, "to="+cubePath, + "csv="+csvfile, "tablename=TestTable"}; + + UserInterface options(APP_XML, args); + try { + csv2table(options); + } + catch (IException &e) { + FAIL() << "Unable to open image: " << e.what() << std::endl; + } + + csvfile = "data/csv2table/test_2.csv"; + args = {"label="+cubePath, "to="+cubePath, + "csv="+csvfile, "tablename=TestTable"}; + options = UserInterface(APP_XML, args); + try { + csv2table(options); + } + catch (IException &e) { + FAIL() << "Unable to open image: " << e.what() << std::endl; + } + + testCube->open(cubePath); + + Table testTable("Temp"); + try { + testTable = testCube->readTable("TestTable"); + } catch(IException &e) { + std::string msg = "Failed to find/read TestTable"; + throw IException(e, IException::Unknown, msg, _FILEINFO_); + } + EXPECT_EQ(testTable.Records(), 3); + EXPECT_EQ(testTable.RecordFields(), 6); + EXPECT_EQ(testTable.RecordSize(), 48); + std::vector expected_values = {10, 100, 11, 0.2, 0.4, -5, + .45, 0.45, -12.58746324, 2, 7, -10, + 3, -1000000, 1000000, 100, 0.45678, 11}; + + for (int i = 0; i < testTable.Records(); i++) { + TableRecord record = testTable[i]; + for (int j = 0; j < record.Fields(); j++) { + EXPECT_EQ(expected_values[(i * record.Fields()) + j], double(record[j])); + } + } +} + +TEST_F(DefaultCube, FunctionalTestCsv2TableErrors) { + QTemporaryDir tempDir; + QString cubePath = testCube->fileName(); + testCube->close(); + QVector args = {"label="+cubePath, "to="+cubePath, + "csv=not_a_file.csv", "tablename=TestTable"}; + + UserInterface options(APP_XML, args); + try { + csv2table(options); + FAIL() << "Csv2table should have failed" << std::endl; + } + catch (IException &e) { + std::cout << e.what() << std::endl; + EXPECT_THAT(e.what(), testing::HasSubstr("Failed to read CSV file")); + EXPECT_THAT(e.what(), testing::HasSubstr("Unable to open file")); + } + + args = {"label="+cubePath, "to="+cubePath, + "csv=data/csv2table/empty.csv", "tablename=TestTable"}; + options = UserInterface(APP_XML, args); + try { + csv2table(options); + FAIL() << "Csv2table should have failed" << std::endl; + } + catch (IException &e) { + EXPECT_THAT(e.what(), testing::HasSubstr("CSV file does not have data")); + EXPECT_THAT(e.what(), testing::HasSubstr("File has [0] rows and [0] columns.")); + } + + args = {"label=not_a_file.pvl", "to="+cubePath, + "csv=data/csv2table/test.csv", "tablename=TestTable"}; + options = UserInterface(APP_XML, args); + try { + csv2table(options); + FAIL() << "Csv2table should have failed" << std::endl; + } + catch (IException &e) { + EXPECT_THAT(e.what(), testing::HasSubstr("Failed to read PVL label file")); + EXPECT_THAT(e.what(), testing::HasSubstr("Unable to open")); + } +} + +TEST_F(DefaultCube, FunctionalTestCsv2TableTypes) { + QTemporaryDir tempDir; + QString csvfile = "data/csv2table/test.csv"; + QString cubePath = testCube->fileName(); + testCube->close(); + QVector args = {"label="+cubePath, "to="+cubePath, + "csv="+csvfile, "tablename=TestTable", "coltypes=(Double,Real,Double,Double,Double,Integer)"}; + + UserInterface options(APP_XML, args); + try { + csv2table(options); + } + catch (IException &e) { + FAIL() << "Unable to open image: " << e.what() << std::endl; + } + + testCube->open(cubePath); + + Table testTable("Temp"); + try { + testTable = testCube->readTable("TestTable"); + } catch(IException &e) { + std::string msg = "Failed to find/read TestTable"; + throw IException(e, IException::Unknown, msg, _FILEINFO_); + } + EXPECT_EQ(testTable.Records(), 3); + EXPECT_EQ(testTable.RecordFields(), 6); + EXPECT_EQ(testTable.RecordSize(), 40); + std::vector expected_values = {10, 0, 11, 0.2, 0.4, -5, + .45, 0.45, -12.58746324, 2, 7, -10, + 3, -1000000, 1000000, 100, 0.45678, 11}; + + for (int i = 0; i < testTable.Records(); i++) { + TableRecord record = testTable[i]; + for (int j = 0; j < record.Fields(); j++) { + if (j == 0 || j == 2 || j == 3 || j == 4) { + EXPECT_EQ(expected_values[(i * record.Fields()) + j], double(record[j])); + } + else if (j == 1) { + EXPECT_EQ((float)expected_values[(i * record.Fields()) + j], float(record[j])); + } + else { + EXPECT_EQ(expected_values[(i * record.Fields()) + j], int(record[j])); + } + } + } +} + +TEST_F(DefaultCube, FunctionalTestCsv2TableArrayTypes) { + QTemporaryDir tempDir; + QString csvfile = "data/csv2table/test_type_arrays.csv"; + QString cubePath = testCube->fileName(); + testCube->close(); + QVector args = {"label="+cubePath, "to="+cubePath, + "csv="+csvfile, "tablename=TestTable", "coltypes=(Integer,Integer,Text,Real,Real,Double,Double)"}; + + UserInterface options(APP_XML, args); + try { + csv2table(options); + } + catch (IException &e) { + FAIL() << "Unable to open image: " << e.what() << std::endl; + } + + testCube->open(cubePath); + + Table testTable("Temp"); + try { + testTable = testCube->readTable("TestTable"); + } catch(IException &e) { + std::string msg = "Failed to find/read TestTable"; + throw IException(e, IException::Unknown, msg, _FILEINFO_); + } + EXPECT_EQ(testTable.Records(), 3); + EXPECT_EQ(testTable.RecordFields(), 4); + EXPECT_EQ(testTable.RecordSize(), 39); + std::vector> truth_ints = {{0, 1}, {2, 3}, {4, 5}}; + std::vector truth_strings = {"Apple", "Banana", "Orange"}; + std::vector> truth_floats = {{0.0001, 0.0002}, {0.0003, 0.0004}, {0.0005, 0.0006}}; + std::vector> truth_doubles = {{0.0000000000001, 0.0000000000002}, {0.0000000000003, 0.0000000000004}, {0.0000000000005, 0.0000000000006}}; + + + for (int i = 0; i < testTable.Records(); i++) { + TableRecord record = testTable[i]; + for (int j = 0; j < record.Fields(); j++) { + if (j == 0) { + EXPECT_TRUE(truth_ints[i] == std::vector(record[j])); + } + else if (j == 1) { + std::cout << truth_strings[i] << ", " << QString(record[j]).toStdString() << std::endl; + EXPECT_TRUE(truth_strings[i] == QString(record[j]).toStdString()); + } + else if (j == 2) { + EXPECT_TRUE(truth_floats[i] == std::vector(record[j])); + } + else { + EXPECT_TRUE(truth_doubles[i] == std::vector(record[j])); + } + } + } +} \ No newline at end of file diff --git a/isis/tests/data/csv2table/empty.csv b/isis/tests/data/csv2table/empty.csv new file mode 100644 index 0000000000..e69de29bb2 diff --git a/isis/tests/data/csv2table/test.csv b/isis/tests/data/csv2table/test.csv new file mode 100644 index 0000000000..cf21702547 --- /dev/null +++ b/isis/tests/data/csv2table/test.csv @@ -0,0 +1,4 @@ +a,b,c,4,5,6 +10,0,11,0.2,0.4,-5 +.45,0.45,-12.58746324,2,7,-10 +3,-1000000,1000000,100,0.45678,11 diff --git a/isis/tests/data/csv2table/test_2.csv b/isis/tests/data/csv2table/test_2.csv new file mode 100644 index 0000000000..66a5d0dd78 --- /dev/null +++ b/isis/tests/data/csv2table/test_2.csv @@ -0,0 +1,4 @@ +a,b,c,4,5,6 +10,100,11,0.2,0.4,-5 +.45,0.45,-12.58746324,2,7,-10 +3,-1000000,1000000,100,0.45678,11 diff --git a/isis/tests/data/csv2table/test_arrays.csv b/isis/tests/data/csv2table/test_arrays.csv new file mode 100644 index 0000000000..4030330346 --- /dev/null +++ b/isis/tests/data/csv2table/test_arrays.csv @@ -0,0 +1,4 @@ +a,b(0),b(1),4,5(0),5(1) +10,0,11,0.2,0.4,-5 +.45,0.45,-12.58746324,2,7,-10 +3,-1000000,1000000,100,0.45678,11 diff --git a/isis/tests/data/csv2table/test_type_arrays.csv b/isis/tests/data/csv2table/test_type_arrays.csv new file mode 100644 index 0000000000..54437877ae --- /dev/null +++ b/isis/tests/data/csv2table/test_type_arrays.csv @@ -0,0 +1,4 @@ +a(0),a(1),b(6),c(0),c(1),d(0),d(1) +0, 1,Apple, 0.0001, 0.0002, 0.0000000000001, 0.0000000000002 +2, 3,Banana, 0.0003, 0.0004, 0.0000000000003, 0.0000000000004 +4, 5,Orange, 0.0005, 0.0006, 0.0000000000005, 0.0000000000006 \ No newline at end of file From 60625af1d4b8333fe1c0dda9506172e9b91d503b Mon Sep 17 00:00:00 2001 From: acpaquette Date: Sun, 8 Dec 2024 11:36:55 -0700 Subject: [PATCH 3/5] Removed old csv2table tests --- .../base/apps/csv2table/tsts/Label/Makefile | 14 -------- isis/src/base/apps/csv2table/tsts/Makefile | 4 --- .../apps/csv2table/tsts/NewTable/Makefile | 19 ----------- .../csv2table/tsts/OverwriteTable/Makefile | 13 -------- .../base/apps/csv2table/tsts/errors/Makefile | 33 ------------------- 5 files changed, 83 deletions(-) delete mode 100644 isis/src/base/apps/csv2table/tsts/Label/Makefile delete mode 100644 isis/src/base/apps/csv2table/tsts/Makefile delete mode 100644 isis/src/base/apps/csv2table/tsts/NewTable/Makefile delete mode 100644 isis/src/base/apps/csv2table/tsts/OverwriteTable/Makefile delete mode 100644 isis/src/base/apps/csv2table/tsts/errors/Makefile diff --git a/isis/src/base/apps/csv2table/tsts/Label/Makefile b/isis/src/base/apps/csv2table/tsts/Label/Makefile deleted file mode 100644 index 48d2f26e42..0000000000 --- a/isis/src/base/apps/csv2table/tsts/Label/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -APPNAME = csv2table - -include $(ISISROOT)/make/isismake.tsts - -commands: - cp $(INPUT)/isisTruth.cub $(OUTPUT)/isisTruth.cub; - $(APPNAME) csv=$(INPUT)/test.csv \ - label=$(INPUT)/label.pvl \ - tablename="TestTable" \ - to=$(OUTPUT)/isisTruth.cub > /dev/null; - catlab from=$(OUTPUT)/isisTruth.cub | \ - sed -n '/Object = Table/,/End_Object/p' > \ - $(OUTPUT)/table.pvl; - rm $(OUTPUT)/isisTruth.cub; diff --git a/isis/src/base/apps/csv2table/tsts/Makefile b/isis/src/base/apps/csv2table/tsts/Makefile deleted file mode 100644 index 46d84c74c2..0000000000 --- a/isis/src/base/apps/csv2table/tsts/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -BLANKS = "%-6s" -LENGTH = "%-40s" - -include $(ISISROOT)/make/isismake.tststree diff --git a/isis/src/base/apps/csv2table/tsts/NewTable/Makefile b/isis/src/base/apps/csv2table/tsts/NewTable/Makefile deleted file mode 100644 index 61f7e8aea0..0000000000 --- a/isis/src/base/apps/csv2table/tsts/NewTable/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -APPNAME = csv2table - -include $(ISISROOT)/make/isismake.tsts - -commands: - cp $(INPUT)/isisTruth.cub $(OUTPUT)/isisTruth.cub; - $(APPNAME) csv=$(INPUT)/test.csv \ - tablename="TestTable" \ - to=$(OUTPUT)/isisTruth.cub > /dev/null; - tabledump from=$(OUTPUT)/isisTruth.cub \ - to=$(OUTPUT)/output.csv \ - NAME="TestTable" > /dev/null; - $(APPNAME) csv=$(INPUT)/test_arrays.csv \ - tablename="TestTableArrays" \ - to=$(OUTPUT)/isisTruth.cub > /dev/null; - tabledump from=$(OUTPUT)/isisTruth.cub \ - to=$(OUTPUT)/output_arrays.csv \ - NAME="TestTableArrays" > /dev/null; - rm $(OUTPUT)/isisTruth.cub; diff --git a/isis/src/base/apps/csv2table/tsts/OverwriteTable/Makefile b/isis/src/base/apps/csv2table/tsts/OverwriteTable/Makefile deleted file mode 100644 index 183ba78dfb..0000000000 --- a/isis/src/base/apps/csv2table/tsts/OverwriteTable/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -APPNAME = csv2table - -include $(ISISROOT)/make/isismake.tsts - -commands: - cp $(INPUT)/isisTruth.cub $(OUTPUT)/isisTruth.cub; - $(APPNAME) csv=$(INPUT)/test.csv \ - tablename="TestTable" \ - to=$(OUTPUT)/isisTruth.cub > /dev/null; - tabledump from=$(OUTPUT)/isisTruth.cub \ - to=$(OUTPUT)/output.csv \ - NAME="TestTable" > /dev/null; - rm $(OUTPUT)/isisTruth.cub; diff --git a/isis/src/base/apps/csv2table/tsts/errors/Makefile b/isis/src/base/apps/csv2table/tsts/errors/Makefile deleted file mode 100644 index ab23c6904b..0000000000 --- a/isis/src/base/apps/csv2table/tsts/errors/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -APPNAME = csv2table - -include $(ISISROOT)/make/isismake.tsts - -commands: - cp $(INPUT)/isisTruth.cub $(OUTPUT)/isisTruth.cub; - - if [ `$(APPNAME) csv=$(INPUT)/not_a_file.csv \ - tablename="TestTable" \ - to=$(OUTPUT)/isisTruth.cub 2> $(OUTPUT)/errors.txt` ]; \ - then \ - true; \ - fi; - - if [ `$(APPNAME) csv=$(INPUT)/empty.csv \ - tablename="TestTable" \ - to=$(OUTPUT)/isisTruth.cub 2>> $(OUTPUT)/errors.txt` ]; \ - then \ - true; \ - fi; - - if [ `$(APPNAME) csv=$(INPUT)/test.csv \ - label=$(INPUT)/not_a_file.pvl \ - tablename="TestTable" \ - to=$(OUTPUT)/isisTruth.cub 2>> $(OUTPUT)/errors.txt` ]; \ - then \ - true; \ - fi; - - cat $(OUTPUT)/errors.txt | sed 's+\[.*input/+[+' \ - > $(OUTPUT)/clean_errors.txt; - - rm $(OUTPUT)/isisTruth.cub $(OUTPUT)/errors.txt; From ca27d235b5235438ca19ebb732f9cbfd136956d4 Mon Sep 17 00:00:00 2001 From: acpaquette Date: Mon, 6 Jan 2025 15:15:35 -0700 Subject: [PATCH 4/5] Added changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1530b840c8..b7b9406401 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ release. - Added majority replacement for reduce app [#5101](https://github.com/DOI-USGS/ISIS3/issues/5101). - Added HRSC support in socetlinescankeywords [#5465](https://github.com/DOI-USGS/ISIS3/issues/5465) - Added option to save and apply bundle adjustment values in `jigsaw` [#4474](https://github.com/DOI-USGS/ISIS3/issues/4474) +- Added the ability to pass column types in csv2table to set the column types in the resulting ISIS table [#5631](https://github.com/DOI-USGS/ISIS3/pull/5631) ### Changed - Refactored the pixel2map app From b615bf20ba5e265a75d3c3eb17b1e0285944d10d Mon Sep 17 00:00:00 2001 From: acpaquette Date: Tue, 7 Jan 2025 11:28:45 -0700 Subject: [PATCH 5/5] Added csv2table files --- isis/src/base/apps/csv2table/csv2table.cpp | 218 +++++++++++++++++++++ isis/src/base/apps/csv2table/csv2table.h | 18 ++ 2 files changed, 236 insertions(+) create mode 100644 isis/src/base/apps/csv2table/csv2table.cpp create mode 100644 isis/src/base/apps/csv2table/csv2table.h diff --git a/isis/src/base/apps/csv2table/csv2table.cpp b/isis/src/base/apps/csv2table/csv2table.cpp new file mode 100644 index 0000000000..8d63215b8e --- /dev/null +++ b/isis/src/base/apps/csv2table/csv2table.cpp @@ -0,0 +1,218 @@ +/** This is free and unencumbered software released into the public domain. + +The authors of ISIS do not claim copyright on the contents of this file. +For more details about the LICENSE terms and the AUTHORS, you will +find files of those names at the top level of this repository. **/ + +/* SPDX-License-Identifier: CC0-1.0 */ +#include "csv2table.h" + +#include + +#include +#include + +#include "Cube.h" +#include "CSVReader.h" +#include "IException.h" +#include "IString.h" +#include "Pvl.h" +#include "PvlObject.h" +#include "Table.h" +#include "TableField.h" +#include "TableRecord.h" + +using namespace std; + +namespace Isis { + /** + * csminit a cube in an Application + * + * @param ui The Application UI + * @param(out) log The Pvl that attempted models will be logged to + */ + void csv2table(UserInterface &ui, Pvl *log) { + // Read the CSV file and get the header + QString csvFileName = ui.GetFileName("csv"); + CSVReader reader; + try { + reader = CSVReader(csvFileName, true); + } + catch(IException &e) { + QString msg = "Failed to read CSV file [" + csvFileName + "]."; + throw IException(e, IException::Io, msg, _FILEINFO_); + } + int numColumns = reader.columns(); + int numRows = reader.rows(); + if (numColumns < 1 || numRows < 1) { + QString msg = "CSV file does not have data.\nFile has [" + toString(numRows) + + "] rows and [" + toString(numColumns) +"] columns."; + throw IException(IException::User, msg, _FILEINFO_); + } + + std::vector fieldTypes; + ui.GetAsString("coltypes", fieldTypes); + + std::vector tableTypes; + if (fieldTypes.size() == 1 && fieldTypes[0] == "") { + for (int i = 0; i < numColumns; i++) { + tableTypes.push_back(TableField::Double); + } + } + else { + if (fieldTypes.size() == numColumns) { + for (QString type: fieldTypes) { + QString upper_type = type.toUpper(); + if (upper_type == "INTEGER") { + tableTypes.push_back(TableField::Type::Integer); + } + else if (upper_type == "DOUBLE") { + tableTypes.push_back(TableField::Type::Double); + } + else if (upper_type == "TEXT") { + tableTypes.push_back(TableField::Type::Text); + } + else if (upper_type == "REAL") { + tableTypes.push_back(TableField::Type::Real); + } + else { + QString msg = "Field [" + type + "] cannot be translated. Accepted types are " + "Integer, Double, Text, and Real"; + throw IException(IException::User, msg, _FILEINFO_); + } + } + } + else { + int numFields = fieldTypes.size(); + QString msg = "Number of fields provided does not equal the number of columns in the CSV. " + "Number of fields [" + toString(numFields) + + "] vs Number of Columns [" + toString(numColumns) + "]"; + throw IException(IException::User, msg, _FILEINFO_); + } + } + + + CSVReader::CSVAxis header = reader.getHeader(); + + // Construct an empty table with the CSV header as field names + // Collect identical field names together, including those with (###) at the end, so a single + // table field with multiple values can be created. + TableRecord tableRow; + QRegularExpression rex(R"((?\w+)(\((?[0-9]*)\)|))"); + for (int columnIndex = 0; columnIndex < numColumns; columnIndex++) { + QRegularExpressionMatch match = rex.match(header[columnIndex]); + if (match.hasMatch()) { + QString name = match.captured("name"); + QString index = match.captured("index"); + + // If the next column header is different, create a field for this one + QRegularExpressionMatch nextMatch = (columnIndex0)?(index.toInt()+1):1); + tableRow += columnField; + } + } + } + + QString tableName = ui.GetString("tablename"); + Table table(tableName, tableRow); + + // Fill the table from the csv + for (int rowIndex = 0; rowIndex < numRows; rowIndex++) { + CSVReader::CSVAxis csvRow = reader.getRow(rowIndex); + for (int columnIndex = 0, fieldIndex = 0; columnIndex < numColumns; ) { + if (tableRow[fieldIndex].size() == 1 || + tableRow[fieldIndex].isText()) { + switch(tableTypes[columnIndex]) { + case TableField::Type::Integer: + tableRow[fieldIndex] = toInt(csvRow[columnIndex++]); + break; + case TableField::Type::Double: + tableRow[fieldIndex] = toDouble(csvRow[columnIndex++]); + break; + case TableField::Type::Text: + tableRow[fieldIndex] = QString(csvRow[columnIndex++]); + break; + case TableField::Type::Real: + tableRow[fieldIndex] = (float)toDouble(csvRow[columnIndex++]); + break; + } + } + else { + std::vector intVector; + std::vector dblVector; + std::vector realVector; + QString strMsg = "TableRecord can't handle list of Strings"; + switch(tableTypes[columnIndex]) { + case TableField::Type::Integer: + for (int arrayLen = 0; arrayLen < tableRow[fieldIndex].size(); arrayLen++) { + intVector.push_back(toInt(csvRow[columnIndex++])); + } + tableRow[fieldIndex] = intVector; + break; + case TableField::Type::Double: + for (int arrayLen = 0; arrayLen < tableRow[fieldIndex].size(); arrayLen++) { + dblVector.push_back(toDouble(csvRow[columnIndex++])); + } + tableRow[fieldIndex] = dblVector; + break; + case TableField::Type::Text: + throw IException(IException::User, strMsg, _FILEINFO_); + break; + case TableField::Type::Real: + for (int arrayLen = 0; arrayLen < tableRow[fieldIndex].size(); arrayLen++) { + realVector.push_back((float)toDouble(csvRow[columnIndex++])); + } + tableRow[fieldIndex] = realVector; + break; + } + intVector.clear(); + dblVector.clear(); + realVector.clear(); + } + fieldIndex++; + } + table += tableRow; + } + + // If a set of additional label keywords was given then add them to the table's pvl description + if (ui.WasEntered("label")) { + QString labelPvlFilename = ui.GetFileName("label"); + Pvl labelPvl; + try { + labelPvl.read(labelPvlFilename); + } + catch(IException &e) { + QString msg = "Failed to read PVL label file [" + labelPvlFilename + "]."; + throw IException(e, IException::Io, msg, _FILEINFO_); + } + + PvlObject &tableLabel = table.Label(); + for (int keyIndex = 0; keyIndex < labelPvl.keywords(); keyIndex++) { + tableLabel.addKeyword(labelPvl[keyIndex]); + } + } + + // Write the table to the cube + QString outCubeFileName(ui.GetCubeName("to")); + Cube outCube; + try { + outCube.open(outCubeFileName, "rw"); + } + catch(IException &e) { + QString msg = "Could not open output cube [" + outCubeFileName + "]."; + throw IException(e, IException::Io, msg, _FILEINFO_); + } + + try { + outCube.write(table); + } + catch(IException &e) { + QString msg = "Could not write output table [" + tableName + + "] to output cube [" + outCubeFileName + "]."; + throw IException(e, IException::Io, msg, _FILEINFO_); + } + + outCube.close(); + } +} \ No newline at end of file diff --git a/isis/src/base/apps/csv2table/csv2table.h b/isis/src/base/apps/csv2table/csv2table.h new file mode 100644 index 0000000000..a2b98cdb76 --- /dev/null +++ b/isis/src/base/apps/csv2table/csv2table.h @@ -0,0 +1,18 @@ +#ifndef csv2table_h +#define csv2table_h +/** This is free and unencumbered software released into the public domain. + +The authors of ISIS do not claim copyright on the contents of this file. +For more details about the LICENSE terms and the AUTHORS, you will +find files of those names at the top level of this repository. **/ + +/* SPDX-License-Identifier: CC0-1.0 */ + +#include "Pvl.h" +#include "UserInterface.h" + +namespace Isis { + extern void csv2table(UserInterface &ui, Pvl *log=nullptr); +} + +#endif \ No newline at end of file