/* -*- mode: java; c-basic-offset: 2; indent-tabs-mode: nil -*- */
/*
Part of the Processing project - http://processing.org
Copyright (c) 2011-13 Ben Fry and Casey Reas
Copyright (c) 2006-11 Ben Fry
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License version 2.1 as published by the Free Software Foundation.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General
Public License along with this library; if not, write to the
Free Software Foundation, Inc., 59 Temple Place, Suite 330,
Boston, MA 02111-1307 USA
*/
package processing.data;
import java.io.*;
import java.lang.reflect.*;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Types;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import javax.xml.parsers.ParserConfigurationException;
import org.xml.sax.SAXException;
import processing.core.PApplet;
import processing.core.PConstants;
/**
* Generic class for handling tabular data, typically from a CSV, TSV, or
* other sort of spreadsheet file.
* CSV files are
* comma separated values,
* often with the data in quotes. TSV files use tabs as separators, and usually
* don't bother with the quotes.
* File names should end with .csv if they're comma separated.
* A rough "spec" for CSV can be found here.
*
* @webref data:composite
* @see PApplet#loadTable(String)
* @see PApplet#saveTable(Table, String)
* @see TableRow
*/
public class Table {
protected int rowCount;
protected int allocCount;
// protected boolean skipEmptyRows = true;
// protected boolean skipCommentLines = true;
// protected String extension = null;
// protected boolean commaSeparatedValues = false;
// protected boolean awfulCSV = false;
protected String missingString = null;
protected int missingInt = 0;
protected long missingLong = 0;
protected float missingFloat = Float.NaN;
protected double missingDouble = Double.NaN;
protected int missingCategory = -1;
String[] columnTitles;
HashMapBlows[] columnCategories;
HashMap columnIndices;
protected Object[] columns; // [column]
// accessible for advanced users
static public final int STRING = 0;
static public final int INT = 1;
static public final int LONG = 2;
static public final int FLOAT = 3;
static public final int DOUBLE = 4;
static public final int CATEGORY = 5;
int[] columnTypes;
protected RowIterator rowIterator;
// 0 for doubling each time, otherwise the number of rows to increment on
// each expansion.
protected int expandIncrement;
/**
* Creates a new, empty table. Use addRow() to add additional rows.
*/
public Table() {
init();
}
/**
* @nowebref
*/
public Table(File file) throws IOException {
this(file, null);
}
/**
* version that uses a File object; future releases (or data types)
* may include additional optimizations here
*
* @nowebref
*/
public Table(File file, String options) throws IOException {
// uses createInput() to handle .gz (and eventually .bz2) files
init();
parse(PApplet.createInput(file),
extensionOptions(true, file.getName(), options));
}
/**
* @nowebref
*/
public Table(InputStream input) throws IOException {
this(input, null);
}
/**
* Read the table from a stream. Possible options include:
*
* - csv - parse the table as comma-separated values
*
- tsv - parse the table as tab-separated values
*
- newlines - this CSV file contains newlines inside individual cells
*
- header - this table has a header (title) row
*
*
* @nowebref
* @param input
* @param options
* @throws IOException
*/
public Table(InputStream input, String options) throws IOException {
init();
parse(input, options);
}
public Table(Iterable rows) {
init();
int row = 0;
int alloc = 10;
for (TableRow incoming : rows) {
if (row == 0) {
setColumnTypes(incoming.getColumnTypes());
setColumnTitles(incoming.getColumnTitles());
// Do this after setting types, otherwise it'll attempt to parse the
// allocated but empty rows, and drive CATEGORY columns nutso.
setRowCount(alloc);
} else if (row == alloc) {
// Far more efficient than re-allocating all columns and doing a copy
alloc *= 2;
setRowCount(alloc);
}
//addRow(row);
setRow(row++, incoming);
}
// Shrink the table to only the rows that were used
if (row != alloc) {
setRowCount(row);
}
}
/**
* @nowebref
*/
public Table(ResultSet rs) {
init();
try {
ResultSetMetaData rsmd = rs.getMetaData();
int columnCount = rsmd.getColumnCount();
setColumnCount(columnCount);
for (int col = 0; col < columnCount; col++) {
setColumnTitle(col, rsmd.getColumnName(col + 1));
int type = rsmd.getColumnType(col + 1);
switch (type) { // TODO these aren't tested. nor are they complete.
case Types.INTEGER:
case Types.TINYINT:
case Types.SMALLINT:
setColumnType(col, INT);
break;
case Types.BIGINT:
setColumnType(col, LONG);
break;
case Types.FLOAT:
setColumnType(col, FLOAT);
break;
case Types.DECIMAL:
case Types.DOUBLE:
case Types.REAL:
setColumnType(col, DOUBLE);
break;
}
}
int row = 0;
while (rs.next()) {
for (int col = 0; col < columnCount; col++) {
switch (columnTypes[col]) {
case STRING: setString(row, col, rs.getString(col+1)); break;
case INT: setInt(row, col, rs.getInt(col+1)); break;
case LONG: setLong(row, col, rs.getLong(col+1)); break;
case FLOAT: setFloat(row, col, rs.getFloat(col+1)); break;
case DOUBLE: setDouble(row, col, rs.getDouble(col+1)); break;
default: throw new IllegalArgumentException("column type " + columnTypes[col] + " not supported.");
}
}
row++;
// String[] row = new String[columnCount];
// for (int col = 0; col < columnCount; col++) {
// row[col] = rs.get(col + 1);
// }
// addRow(row);
}
} catch (SQLException s) {
throw new RuntimeException(s);
}
}
public Table typedParse(InputStream input, String options) throws IOException {
Table table = new Table();
table.setColumnTypes(this);
table.parse(input, options);
return table;
}
protected void init() {
columns = new Object[0];
columnTypes = new int[0];
columnCategories = new HashMapBlows[0];
}
/*
protected String checkOptions(File file, String options) throws IOException {
String extension = null;
String filename = file.getName();
int dotIndex = filename.lastIndexOf('.');
if (dotIndex != -1) {
extension = filename.substring(dotIndex + 1).toLowerCase();
if (!extension.equals("csv") &&
!extension.equals("tsv") &&
!extension.equals("html") &&
!extension.equals("bin")) {
// ignore extension
extension = null;
}
}
if (extension == null) {
if (options == null) {
throw new IOException("This table filename has no extension, and no options are set.");
}
} else { // extension is not null
if (options == null) {
options = extension;
} else {
// prepend the extension, it will be overridden if there's an option for it.
options = extension + "," + options;
}
}
return options;
}
*/
static final String[] loadExtensions = { "csv", "tsv", "ods", "bin" };
static final String[] saveExtensions = { "csv", "tsv", "html", "bin" };
static public String extensionOptions(boolean loading, String filename, String options) {
String extension = PApplet.checkExtension(filename);
if (extension != null) {
for (String possible : loading ? loadExtensions : saveExtensions) {
if (extension.equals(possible)) {
if (options == null) {
return extension;
} else {
// prepend the extension to the options (will be replaced by other
// options that override it later in the load loop)
return extension + "," + options;
}
}
}
}
return options;
}
protected void parse(InputStream input, String options) throws IOException {
//init();
boolean awfulCSV = false;
boolean header = false;
String extension = null;
boolean binary = false;
String worksheet = null;
final String sheetParam = "worksheet=";
String[] opts = null;
if (options != null) {
opts = PApplet.trim(PApplet.split(options, ','));
for (String opt : opts) {
if (opt.equals("tsv")) {
extension = "tsv";
} else if (opt.equals("csv")) {
extension = "csv";
} else if (opt.equals("ods")) {
extension = "ods";
} else if (opt.equals("newlines")) {
awfulCSV = true;
extension = "csv";
} else if (opt.equals("bin")) {
binary = true;
extension = "bin";
} else if (opt.equals("header")) {
header = true;
} else if (opt.startsWith(sheetParam)) {
worksheet = opt.substring(sheetParam.length());
} else if (opt.startsWith("dictionary=")) {
// ignore option, this is only handled by PApplet
} else {
throw new IllegalArgumentException("'" + opt + "' is not a valid option for loading a Table");
}
}
}
if (extension == null) {
throw new IllegalArgumentException("No extension specified for this Table");
}
if (binary) {
loadBinary(input);
} else if (extension.equals("ods")) {
odsParse(input, worksheet);
} else {
BufferedReader reader = PApplet.createReader(input);
if (awfulCSV) {
parseAwfulCSV(reader, header);
} else if ("tsv".equals(extension)) {
parseBasic(reader, header, true);
} else if ("csv".equals(extension)) {
parseBasic(reader, header, false);
}
}
}
protected void parseBasic(BufferedReader reader,
boolean header, boolean tsv) throws IOException {
String line = null;
int row = 0;
if (rowCount == 0) {
setRowCount(10);
}
//int prev = 0; //-1;
try {
while ((line = reader.readLine()) != null) {
if (row == getRowCount()) {
setRowCount(row