You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
tellico/src/translators/csvimporter.cpp

579 lines
19 KiB

/***************************************************************************
copyright : (C) 2003-2006 by Robby Stephenson
email : robby@periapsis.org
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of version 2 of the GNU General Public License as *
* published by the Free Software Foundation; *
* *
***************************************************************************/
#include "csvimporter.h"
#include "translators.h" // needed for ImportAction
#include "../collectionfieldsdialog.h"
#include "../document.h"
#include "../collection.h"
#include "../progressmanager.h"
#include "../tellico_debug.h"
#include "../collectionfactory.h"
#include "../gui/collectiontypecombo.h"
#include "../latin1literal.h"
#include "../stringset.h"
extern "C" {
#include "libcsv.h"
}
#include <klineedit.h>
#include <kcombobox.h>
#include <knuminput.h>
#include <kpushbutton.h>
#include <tdeapplication.h>
#include <kiconloader.h>
#include <tdeconfig.h>
#include <tdemessagebox.h>
#include <tqgroupbox.h>
#include <tqlayout.h>
#include <tqhbox.h>
#include <tqlabel.h>
#include <tqcheckbox.h>
#include <tqbuttongroup.h>
#include <tqradiobutton.h>
#include <tqwhatsthis.h>
#include <tqtable.h>
#include <tqvaluevector.h>
#include <tqregexp.h>
using Tellico::Import::CSVImporter;
typedef int(*SpaceFunc)(char);
static void writeToken(char* buffer, size_t len, void* data);
static void writeRow(char buffer, void* data);
static int isSpace(char c);
static int isSpaceOrTab(char c);
static int isTab(char c);
class CSVImporter::Parser {
public:
Parser(const TQString& str) : stream(new TQTextIStream(&str)) { csv_init(&parser, 0); }
~Parser() { csv_free(parser); delete stream; stream = 0; }
void setDelimiter(const TQString& s) {
Q_ASSERT(s.length() == 1);
csv_set_delim(parser, s[0].latin1());
if(s[0] == '\t') csv_set_space_func(parser, isSpace);
else if(s[0] == ' ') csv_set_space_func(parser, isTab);
else csv_set_space_func(parser, isSpaceOrTab);
}
void reset(const TQString& str) { delete stream; stream = new TQTextIStream(&str); };
bool hasNext() { return !stream->atEnd(); }
void skipLine() { stream->readLine(); }
void addToken(const TQString& t) { tokens += t; }
void setRowDone(bool b) { done = b; }
TQStringList nextTokens() {
tokens.clear();
done = false;
while(hasNext() && !done) {
TQCString line = stream->readLine().utf8() + '\n'; // need the eol char
csv_parse(parser, line, line.length(), &writeToken, &writeRow, this);
}
csv_fini(parser, &writeToken, &writeRow, this);
return tokens;
}
private:
struct csv_parser* parser;
TQTextIStream* stream;
TQStringList tokens;
bool done;
};
static void writeToken(char* buffer, size_t len, void* data) {
CSVImporter::Parser* p = static_cast<CSVImporter::Parser*>(data);
p->addToken(TQString::fromUtf8(buffer, len));
}
static void writeRow(char c, void* data) {
Q_UNUSED(c);
CSVImporter::Parser* p = static_cast<CSVImporter::Parser*>(data);
p->setRowDone(true);
}
static int isSpace(char c) {
if (c == CSV_SPACE) return 1;
return 0;
}
static int isSpaceOrTab(char c) {
if (c == CSV_SPACE || c == CSV_TAB) return 1;
return 0;
}
static int isTab(char c) {
if (c == CSV_TAB) return 1;
return 0;
}
CSVImporter::CSVImporter(const KURL& url_) : Tellico::Import::TextImporter(url_),
m_coll(0),
m_existingCollection(0),
m_firstRowHeader(false),
m_delimiter(TQString::fromLatin1(",")),
m_cancelled(false),
m_widget(0),
m_table(0),
m_hasAssignedFields(false),
m_parser(new Parser(text())) {
m_parser->setDelimiter(m_delimiter);
}
CSVImporter::~CSVImporter() {
delete m_parser;
m_parser = 0;
}
Tellico::Data::CollPtr CSVImporter::collection() {
// don't just check if m_coll is non-null since the collection can be created elsewhere
if(m_coll && m_coll->entryCount() > 0) {
return m_coll;
}
if(!m_coll) {
m_coll = CollectionFactory::collection(m_comboColl->currentType(), true);
}
const TQStringList existingNames = m_coll->fieldNames();
TQValueVector<int> cols;
TQStringList names;
for(int col = 0; col < m_table->numCols(); ++col) {
TQString t = m_table->horizontalHeader()->label(col);
if(m_existingCollection && m_existingCollection->fieldByTitle(t)) {
// the collection might have the right field, but a different title, say for translations
Data::FieldPtr f = m_existingCollection->fieldByTitle(t);
if(m_coll->hasField(f->name())) {
// might have different values settings
m_coll->removeField(f->name(), true /* force */);
}
m_coll->addField(new Data::Field(*f));
cols.push_back(col);
names << f->name();
} else if(m_coll->fieldByTitle(t)) {
cols.push_back(col);
names << m_coll->fieldNameByTitle(t);
}
}
if(names.isEmpty()) {
myDebug() << "CSVImporter::collection() - no fields assigned" << endl;
return 0;
}
m_parser->reset(text());
// if the first row are headers, skip it
if(m_firstRowHeader) {
m_parser->skipLine();
}
const uint numLines = text().contains('\n');
const uint stepSize = TQMAX(s_stepSize, numLines/100);
const bool showProgress = options() & ImportProgress;
ProgressItem& item = ProgressManager::self()->newProgressItem(this, progressLabel(), true);
item.setTotalSteps(numLines);
connect(&item, TQT_SIGNAL(signalCancelled(ProgressItem*)), TQT_SLOT(slotCancel()));
ProgressItem::Done done(this);
uint j = 0;
while(!m_cancelled && m_parser->hasNext()) {
bool empty = true;
Data::EntryPtr entry = new Data::Entry(m_coll);
TQStringList values = m_parser->nextTokens();
for(uint i = 0; i < names.size(); ++i) {
// TQString value = values[cols[i]].simplifyWhiteSpace();
TQString value = values[cols[i]].stripWhiteSpace();
bool success = entry->setField(names[i], value);
// we might need to add a new allowed value
// assume that if the user is importing the value, it should be allowed
if(!success && m_coll->fieldByName(names[i])->type() == Data::Field::Choice) {
Data::FieldPtr f = m_coll->fieldByName(names[i]);
StringSet allow;
allow.add(f->allowed());
allow.add(value);
f->setAllowed(allow.toList());
m_coll->modifyField(f);
success = entry->setField(names[i], value);
}
if(empty && success) {
empty = false;
}
}
if(!empty) {
m_coll->addEntries(entry);
}
if(showProgress && j%stepSize == 0) {
ProgressManager::self()->setProgress(this, j);
kapp->processEvents();
}
++j;
}
{
TDEConfigGroup config(TDEGlobal::config(), TQString::fromLatin1("ImportOptions - CSV"));
config.writeEntry("Delimiter", m_delimiter);
config.writeEntry("First Row Titles", m_firstRowHeader);
}
return m_coll;
}
TQWidget* CSVImporter::widget(TQWidget* parent_, const char* name_) {
if(m_widget && m_widget->parent() == parent_) {
return m_widget;
}
m_widget = new TQWidget(parent_, name_);
TQVBoxLayout* l = new TQVBoxLayout(m_widget);
TQGroupBox* group = new TQGroupBox(1, Qt::Horizontal, i18n("CSV Options"), m_widget);
l->addWidget(group);
TQHBox* box = new TQHBox(group);
box->setSpacing(5);
TQLabel* lab = new TQLabel(i18n("Collection &type:"), box);
m_comboColl = new GUI::CollectionTypeCombo(box);
lab->setBuddy(m_comboColl);
TQWhatsThis::add(m_comboColl, i18n("Select the type of collection being imported."));
connect(m_comboColl, TQT_SIGNAL(activated(int)), TQT_SLOT(slotTypeChanged()));
// need a spacer
TQWidget* w = new TQWidget(box);
box->setStretchFactor(w, 1);
m_checkFirstRowHeader = new TQCheckBox(i18n("&First row contains field titles"), group);
TQWhatsThis::add(m_checkFirstRowHeader, i18n("If checked, the first row is used as field titles."));
connect(m_checkFirstRowHeader, TQT_SIGNAL(toggled(bool)), TQT_SLOT(slotFirstRowHeader(bool)));
TQHBox* hbox2 = new TQHBox(group);
m_delimiterGroup = new TQButtonGroup(0, Qt::Vertical, i18n("Delimiter"), hbox2);
TQGridLayout* m_delimiterGroupLayout = new TQGridLayout(m_delimiterGroup->layout(), 3, 3);
m_delimiterGroupLayout->setAlignment(TQt::AlignTop);
TQWhatsThis::add(m_delimiterGroup, i18n("In addition to a comma, other characters may be used as "
"a delimiter, separating each value in the file."));
connect(m_delimiterGroup, TQT_SIGNAL(clicked(int)), TQT_SLOT(slotDelimiter()));
m_radioComma = new TQRadioButton(m_delimiterGroup);
m_radioComma->setText(i18n("&Comma"));
m_radioComma->setChecked(true);
TQWhatsThis::add(m_radioComma, i18n("Use a comma as the delimiter."));
m_delimiterGroupLayout->addWidget(m_radioComma, 1, 0);
m_radioSemicolon = new TQRadioButton( m_delimiterGroup);
m_radioSemicolon->setText(i18n("&Semicolon"));
TQWhatsThis::add(m_radioSemicolon, i18n("Use a semi-colon as the delimiter."));
m_delimiterGroupLayout->addWidget(m_radioSemicolon, 1, 1);
m_radioTab = new TQRadioButton(m_delimiterGroup);
m_radioTab->setText(i18n("Ta&b"));
TQWhatsThis::add(m_radioTab, i18n("Use a tab as the delimiter."));
m_delimiterGroupLayout->addWidget(m_radioTab, 2, 0);
m_radioOther = new TQRadioButton(m_delimiterGroup);
m_radioOther->setText(i18n("Ot&her:"));
TQWhatsThis::add(m_radioOther, i18n("Use a custom string as the delimiter."));
m_delimiterGroupLayout->addWidget(m_radioOther, 2, 1);
m_editOther = new KLineEdit(m_delimiterGroup);
m_editOther->setEnabled(false);
m_editOther->setFixedWidth(m_widget->fontMetrics().width('X') * 4);
m_editOther->setMaxLength(1);
TQWhatsThis::add(m_editOther, i18n("A custom string, such as a colon, may be used as a delimiter."));
m_delimiterGroupLayout->addWidget(m_editOther, 2, 2);
connect(m_radioOther, TQT_SIGNAL(toggled(bool)),
m_editOther, TQT_SLOT(setEnabled(bool)));
connect(m_editOther, TQT_SIGNAL(textChanged(const TQString&)), TQT_SLOT(slotDelimiter()));
w = new TQWidget(hbox2);
hbox2->setStretchFactor(w, 1);
m_table = new TQTable(5, 0, group);
m_table->setSelectionMode(TQTable::Single);
m_table->setFocusStyle(TQTable::FollowStyle);
m_table->setLeftMargin(0);
m_table->verticalHeader()->hide();
m_table->horizontalHeader()->setClickEnabled(true);
m_table->setReadOnly(true);
m_table->setMinimumHeight(m_widget->fontMetrics().lineSpacing() * 8);
TQWhatsThis::add(m_table, i18n("The table shows up to the first five lines of the CSV file."));
connect(m_table, TQT_SIGNAL(currentChanged(int, int)), TQT_SLOT(slotCurrentChanged(int, int)));
connect(m_table->horizontalHeader(), TQT_SIGNAL(clicked(int)), TQT_SLOT(slotHeaderClicked(int)));
TQWidget* hbox = new TQWidget(group);
TQHBoxLayout* hlay = new TQHBoxLayout(hbox, 5);
hlay->addStretch(10);
TQWhatsThis::add(hbox, i18n("<qt>Set each column to correspond to a field in the collection by choosing "
"a column, selecting the field, then clicking the <i>Assign Field</i> button.</qt>"));
lab = new TQLabel(i18n("Co&lumn:"), hbox);
hlay->addWidget(lab);
m_colSpinBox = new KIntSpinBox(hbox);
hlay->addWidget(m_colSpinBox);
m_colSpinBox->setMinValue(1);
connect(m_colSpinBox, TQT_SIGNAL(valueChanged(int)), TQT_SLOT(slotSelectColumn(int)));
lab->setBuddy(m_colSpinBox);
hlay->addSpacing(10);
lab = new TQLabel(i18n("&Data field in this column:"), hbox);
hlay->addWidget(lab);
m_comboField = new KComboBox(hbox);
hlay->addWidget(m_comboField);
connect(m_comboField, TQT_SIGNAL(activated(int)), TQT_SLOT(slotFieldChanged(int)));
lab->setBuddy(m_comboField);
hlay->addSpacing(10);
m_setColumnBtn = new KPushButton(i18n("&Assign Field"), hbox);
hlay->addWidget(m_setColumnBtn);
m_setColumnBtn->setIconSet(SmallIconSet(TQString::fromLatin1("apply")));
connect(m_setColumnBtn, TQT_SIGNAL(clicked()), TQT_SLOT(slotSetColumnTitle()));
hlay->addStretch(10);
l->addStretch(1);
TDEConfigGroup config(TDEGlobal::config(), TQString::fromLatin1("ImportOptions - CSV"));
m_delimiter = config.readEntry("Delimiter", m_delimiter);
m_firstRowHeader = config.readBoolEntry("First Row Titles", m_firstRowHeader);
m_checkFirstRowHeader->setChecked(m_firstRowHeader);
if(m_delimiter == Latin1Literal(",")) {
m_radioComma->setChecked(true);
slotDelimiter(); // since the comma box was already checked, the slot won't fire
} else if(m_delimiter == Latin1Literal(";")) {
m_radioSemicolon->setChecked(true);
} else if(m_delimiter == Latin1Literal("\t")) {
m_radioTab->setChecked(true);
} else if(!m_delimiter.isEmpty()) {
m_radioOther->setChecked(true);
m_editOther->setEnabled(true);
m_editOther->setText(m_delimiter);
}
return m_widget;
}
bool CSVImporter::validImport() const {
// at least one column has to be defined
if(!m_hasAssignedFields) {
KMessageBox::sorry(m_widget, i18n("At least one column must be assigned to a field. "
"Only assigned columns will be imported."));
}
return m_hasAssignedFields;
}
void CSVImporter::fillTable() {
if(!m_table) {
return;
}
m_parser->reset(text());
// not skipping first row since the updateHeader() call depends on it
int maxCols = 0;
int row = 0;
for( ; m_parser->hasNext() && row < m_table->numRows(); ++row) {
TQStringList values = m_parser->nextTokens();
if(static_cast<int>(values.count()) > m_table->numCols()) {
m_table->setNumCols(values.count());
m_colSpinBox->setMaxValue(values.count());
}
int col = 0;
for(TQStringList::ConstIterator it = values.begin(); it != values.end(); ++it) {
m_table->setText(row, col, *it);
m_table->adjustColumn(col);
++col;
}
if(col > maxCols) {
maxCols = col;
}
}
for( ; row < m_table->numRows(); ++row) {
for(int col = 0; col < m_table->numCols(); ++col) {
m_table->clearCell(row, col);
}
}
m_table->setNumCols(maxCols);
}
void CSVImporter::slotTypeChanged() {
// iterate over the collection names until it matches the text of the combo box
Data::Collection::Type type = static_cast<Data::Collection::Type>(m_comboColl->currentType());
m_coll = CollectionFactory::collection(type, true);
updateHeader(true);
m_comboField->clear();
m_comboField->insertStringList(m_existingCollection ? m_existingCollection->fieldTitles() : m_coll->fieldTitles());
m_comboField->insertItem('<' + i18n("New Field") + '>');
// hack to force a resize
m_comboField->setFont(m_comboField->font());
m_comboField->updateGeometry();
}
void CSVImporter::slotFirstRowHeader(bool b_) {
m_firstRowHeader = b_;
updateHeader(false);
fillTable();
}
void CSVImporter::slotDelimiter() {
if(m_radioComma->isChecked()) {
m_delimiter = ',';
} else if(m_radioSemicolon->isChecked()) {
m_delimiter = ';';
} else if(m_radioTab->isChecked()) {
m_delimiter = '\t';
} else {
m_editOther->setFocus();
m_delimiter = m_editOther->text();
}
if(!m_delimiter.isEmpty()) {
m_parser->setDelimiter(m_delimiter);
fillTable();
updateHeader(false);
}
}
void CSVImporter::slotCurrentChanged(int, int col_) {
int pos = col_+1;
m_colSpinBox->setValue(pos); //slotSelectColumn() gets called because of the signal
}
void CSVImporter::slotHeaderClicked(int col_) {
int pos = col_+1;
m_colSpinBox->setValue(pos); //slotSelectColumn() gets called because of the signal
}
void CSVImporter::slotSelectColumn(int pos_) {
// pos is really the number of the position of the column
int col = pos_ - 1;
m_table->ensureCellVisible(0, col);
m_comboField->setCurrentItem(m_table->horizontalHeader()->label(col));
}
void CSVImporter::slotSetColumnTitle() {
int col = m_colSpinBox->value()-1;
const TQString title = m_comboField->currentText();
m_table->horizontalHeader()->setLabel(col, title);
m_hasAssignedFields = true;
// make sure none of the other columns have this title
bool found = false;
for(int i = 0; i < col; ++i) {
if(m_table->horizontalHeader()->label(i) == title) {
m_table->horizontalHeader()->setLabel(i, TQString::number(i+1));
found = true;
break;
}
}
// if found, then we're done
if(found) {
return;
}
for(int i = col+1; i < m_table->numCols(); ++i) {
if(m_table->horizontalHeader()->label(i) == title) {
m_table->horizontalHeader()->setLabel(i, TQString::number(i+1));
break;
}
}
}
void CSVImporter::updateHeader(bool force_) {
if(!m_table) {
return;
}
if(!m_firstRowHeader && !force_) {
return;
}
Data::CollPtr c = m_existingCollection ? m_existingCollection : m_coll;
for(int col = 0; col < m_table->numCols(); ++col) {
TQString s = m_table->text(0, col);
Data::FieldPtr f;
if(c) {
c->fieldByTitle(s);
if(!f) {
f = c->fieldByName(s);
}
}
if(m_firstRowHeader && !s.isEmpty() && c && f) {
m_table->horizontalHeader()->setLabel(col, f->title());
m_hasAssignedFields = true;
} else {
m_table->horizontalHeader()->setLabel(col, TQString::number(col+1));
}
}
}
void CSVImporter::slotFieldChanged(int idx_) {
// only care if it's the last item -> add new field
if(idx_ < m_comboField->count()-1) {
return;
}
Data::CollPtr c = m_existingCollection ? m_existingCollection : m_coll;
uint count = c->fieldTitles().count();
CollectionFieldsDialog dlg(c, m_widget);
// dlg.setModal(true);
if(dlg.exec() == TQDialog::Accepted) {
m_comboField->clear();
m_comboField->insertStringList(c->fieldTitles());
m_comboField->insertItem('<' + i18n("New Field") + '>');
if(count != c->fieldTitles().count()) {
fillTable();
}
m_comboField->setCurrentItem(0);
}
}
void CSVImporter::slotActionChanged(int action_) {
Data::CollPtr currColl = Data::Document::self()->collection();
if(!currColl) {
m_existingCollection = 0;
return;
}
switch(action_) {
case Import::Replace:
{
int currType = m_comboColl->currentType();
m_comboColl->reset();
m_comboColl->setCurrentType(currType);
m_existingCollection = 0;
}
break;
case Import::Append:
case Import::Merge:
{
m_comboColl->clear();
TQString name = CollectionFactory::nameMap()[currColl->type()];
m_comboColl->insertItem(name, currColl->type());
m_existingCollection = currColl;
}
break;
}
slotTypeChanged();
}
void CSVImporter::slotCancel() {
m_cancelled = true;
}
#include "csvimporter.moc"