tellico/src/fetch/srufetcher.cpp

/***************************************************************************
    copyright            : (C) 2003-2006 by Robby Stephenson
    email                : robby@periapsis.org
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of version 2 of the GNU General Public License as  *
 *   published by the Free Software Foundation;                            *
 *                                                                         *
 ***************************************************************************/

#include "srufetcher.h"
#include "messagehandler.h"
#include "../field.h"
#include "../collection.h"
#include "../translators/tellico_xml.h"
#include "../translators/xslthandler.h"
#include "../translators/tellicoimporter.h"
#include "../translators/dcimporter.h"
#include "../tellico_kernel.h"
#include "../tellico_debug.h"
#include "../gui/lineedit.h"
#include "../gui/combobox.h"
#include "../latin1literal.h"
#include "../tellico_utils.h"
#include "../lccnvalidator.h"

#include <klocale.h>
#include <kio/job.h>
#include <kstandarddirs.h>
#include <kconfig.h>
#include <kcombobox.h>
#include <kaccelmanager.h>
#include <knuminput.h>

#include <tqlabel.h>
#include <tqlayout.h>
#include <tqwhatsthis.h>

//#define SRU_DEBUG

namespace {
  // 7090 was the old default port, but that wa sjust because LoC used it
  // let's use default HTTP port of 80 now
  static const int SRU_DEFAULT_PORT = 80;
  static const int SRU_MAX_RECORDS = 25;
}

using Tellico::Fetch::SRUFetcher;
using Tellico::Fetch::SRUConfigWidget;

SRUFetcher::SRUFetcher(TQObject* parent_, const char* name_)
    : Fetcher(parent_, name_), m_job(0), m_MARCXMLHandler(0), m_MODSHandler(0), m_started(false) {
}

SRUFetcher::SRUFetcher(const TQString& name_, const TQString& host_, uint port_, const TQString& path_,
                       TQObject* parent_) : Fetcher(parent_),
      m_host(host_), m_port(port_), m_path(path_),
      m_job(0), m_MARCXMLHandler(0), m_MODSHandler(0), m_started(false) {
  m_name = name_; // m_name is protected in super class
}

SRUFetcher::~SRUFetcher() {
  delete m_MARCXMLHandler;
  m_MARCXMLHandler = 0;
  delete m_MODSHandler;
  m_MODSHandler = 0;
}

TQString SRUFetcher::defaultName() {
  return i18n("SRU Server");
}

TQString SRUFetcher::source() const {
  return m_name.isEmpty() ? defaultName() : m_name;
}

bool SRUFetcher::canFetch(int type) const {
  return type == Data::Collection::Book || type == Data::Collection::Bibtex;
}

void SRUFetcher::readConfigHook(const KConfigGroup& config_) {
  m_host = config_.readEntry("Host");
  int p = config_.readNumEntry("Port", SRU_DEFAULT_PORT);
  if(p > 0) {
    m_port = p;
  }
  m_path = config_.readEntry("Path");
  // used to be called Database
  if(m_path.isEmpty()) {
    m_path = config_.readEntry("Database");
  }
  if(!m_path.startsWith(TQChar('/'))) {
    m_path.prepend('/');
  }
  m_format = config_.readEntry("Format", TQString::fromLatin1("mods"));
  m_fields = config_.readListEntry("Custom Fields");
}

void SRUFetcher::search(FetchKey key_, const TQString& value_) {
  if(m_host.isEmpty() || m_path.isEmpty()) {
    myDebug() << "SRUFetcher::search() - settings are not set!" << endl;
    stop();
    return;
  }

  m_started = true;

#ifdef SRU_DEBUG
  KURL u = KURL::fromPathOrURL(TQString::fromLatin1("/home/robby/sru.xml"));
#else
  KURL u;
  u.setProtocol(TQString::fromLatin1("http"));
  u.setHost(m_host);
  u.setPort(m_port);
  u.setPath(m_path);

  u.addQueryItem(TQString::fromLatin1("operation"), TQString::fromLatin1("searchRetrieve"));
  u.addQueryItem(TQString::fromLatin1("version"), TQString::fromLatin1("1.1"));
  u.addQueryItem(TQString::fromLatin1("maximumRecords"), TQString::number(SRU_MAX_RECORDS));
  u.addQueryItem(TQString::fromLatin1("recordSchema"), m_format);

  const int type = Kernel::self()->collectionType();
  TQString str = TQChar('"') + value_ + TQChar('"');
  switch(key_) {
    case Title:
      u.addQueryItem(TQString::fromLatin1("query"), TQString::fromLatin1("dc.title=") + str);
      break;

    case Person:
      {
        TQString s;
        if(type == Data::Collection::Book || type == Data::Collection::Bibtex) {
          s = TQString::fromLatin1("author=") + str + TQString::fromLatin1(" or dc.author=") + str;
        } else {
          s = TQString::fromLatin1("dc.creator=") + str + TQString::fromLatin1(" or dc.editor=") + str;
        }
        u.addQueryItem(TQString::fromLatin1("query"), s);
      }
      break;

    case ISBN:
      // no validation here
      str.remove('-');
      // limit to first isbn
      str = str.section(';', 0, 0);
      u.addQueryItem(TQString::fromLatin1("query"), TQString::fromLatin1("bath.isbn=") + str);
      break;

    case LCCN:
      {
        // limit to first lccn
        str.remove('-');
        str = str.section(';', 0, 0);
        // also try formalized lccn
        TQString lccn = LCCNValidator::formalize(str);
        u.addQueryItem(TQString::fromLatin1("query"),
                       TQString::fromLatin1("bath.lccn=") + str +
                       TQString::fromLatin1(" or bath.lccn=") + lccn
                       );
      }
      break;

    case Keyword:
      u.addQueryItem(TQString::fromLatin1("query"), str);
      break;

    case Raw:
      {
        TQString key = value_.section('=', 0, 0).stripWhiteSpace();
        TQString str = value_.section('=', 1).stripWhiteSpace();
        u.addQueryItem(key, str);
      }
      break;

    default:
      kdWarning() << "SRUFetcher::search() - key not recognized: " << key_ << endl;
      stop();
      break;
  }
#endif
//  myDebug() << u.prettyURL() << endl;

  m_job = KIO::get(u, false, false);
  connect(m_job, TQT_SIGNAL(data(KIO::Job*, const TQByteArray&)),
          TQT_SLOT(slotData(KIO::Job*, const TQByteArray&)));
  connect(m_job, TQT_SIGNAL(result(KIO::Job*)),
          TQT_SLOT(slotComplete(KIO::Job*)));
}

void SRUFetcher::stop() {
  if(!m_started) {
    return;
  }
  if(m_job) {
    m_job->kill();
    m_job = 0;
  }
  m_data.truncate(0);
  m_started = false;
  emit signalDone(this);
}

void SRUFetcher::slotData(KIO::Job*, const TQByteArray& data_) {
  TQDataStream stream(m_data, IO_WriteOnly | IO_Append);
  stream.writeRawBytes(data_.data(), data_.size());
}

void SRUFetcher::slotComplete(KIO::Job* job_) {
  // since the fetch is done, don't worry about holding the job pointer
  m_job = 0;

  if(job_->error()) {
    job_->showErrorDialog(Kernel::self()->widget());
    stop();
    return;
  }

  if(m_data.isEmpty()) {
    stop();
    return;
  }

  Data::CollPtr coll;
  TQString msg;

  const TQString result = TQString::fromUtf8(m_data, m_data.size());

  // first check for SRU errors
  const TQString& diag = XML::nsZingDiag;
  Import::XMLImporter xmlImporter(result);
  TQDomDocument dom = xmlImporter.domDocument();

  TQDomNodeList diagList = dom.elementsByTagNameNS(diag, TQString::fromLatin1("diagnostic"));
  for(uint i = 0; i < diagList.count(); ++i) {
    TQDomElement elem = diagList.item(i).toElement();
    TQDomNodeList nodeList1 = elem.elementsByTagNameNS(diag, TQString::fromLatin1("message"));
    TQDomNodeList nodeList2 = elem.elementsByTagNameNS(diag, TQString::fromLatin1("details"));
    for(uint j = 0; j < nodeList1.count(); ++j) {
      TQString d = nodeList1.item(j).toElement().text();
      if(!d.isEmpty()) {
        TQString d2 = nodeList2.item(j).toElement().text();
        if(!d2.isEmpty()) {
          d += " (" + d2 + ')';
        }
        myDebug() << "SRUFetcher::slotComplete() - " << d << endl;
        if(!msg.isEmpty()) msg += '\n';
        msg += d;
      }
    }
  }

  TQString modsResult;
  if(m_format == Latin1Literal("mods")) {
    modsResult = result;
  } else if(m_format == Latin1Literal("marcxml") && initMARCXMLHandler()) {
    modsResult = m_MARCXMLHandler->applyStylesheet(result);
  }
  if(!modsResult.isEmpty() && initMODSHandler()) {
    Import::TellicoImporter imp(m_MODSHandler->applyStylesheet(modsResult));
    coll = imp.collection();
    if(!msg.isEmpty()) msg += '\n';
    msg += imp.statusMessage();
  } else if(m_format == Latin1Literal("dc")) {
    Import::DCImporter imp(dom);
    coll = imp.collection();
    if(!msg.isEmpty()) msg += '\n';
    msg += imp.statusMessage();
  } else {
    myDebug() << "SRUFetcher::slotComplete() - unrecognized format: " << m_format << endl;
    stop();
    return;
  }

  if(coll && !msg.isEmpty()) {
    message(msg, coll->entryCount() == 0 ? MessageHandler::Warning : MessageHandler::Status);
  }

  if(!coll) {
    myDebug() << "SRUFetcher::slotComplete() - no collection pointer" << endl;
    if(!msg.isEmpty()) {
      message(msg, MessageHandler::Error);
    }
    stop();
    return;
  }

  const StringMap customFields = SRUFetcher::customFields();
  for(StringMap::ConstIterator it = customFields.begin(); it != customFields.end(); ++it) {
    if(!m_fields.contains(it.key())) {
      coll->removeField(it.key());
    }
  }

  Data::EntryVec entries = coll->entries();
  for(Data::EntryVec::Iterator entry = entries.begin(); entry != entries.end(); ++entry) {
    TQString desc;
    switch(coll->type()) {
      case Data::Collection::Book:
        desc = entry->field(TQString::fromLatin1("author"))
               + TQChar('/')
               + entry->field(TQString::fromLatin1("publisher"));
        if(!entry->field(TQString::fromLatin1("cr_year")).isEmpty()) {
          desc += TQChar('/') + entry->field(TQString::fromLatin1("cr_year"));
        } else if(!entry->field(TQString::fromLatin1("pub_year")).isEmpty()){
          desc += TQChar('/') + entry->field(TQString::fromLatin1("pub_year"));
        }
        break;

      case Data::Collection::Video:
        desc = entry->field(TQString::fromLatin1("studio"))
               + TQChar('/')
               + entry->field(TQString::fromLatin1("director"))
               + TQChar('/')
               + entry->field(TQString::fromLatin1("year"));
        break;

      case Data::Collection::Album:
        desc = entry->field(TQString::fromLatin1("artist"))
               + TQChar('/')
               + entry->field(TQString::fromLatin1("label"))
               + TQChar('/')
               + entry->field(TQString::fromLatin1("year"));
        break;

      default:
        break;
    }
    SearchResult* r = new SearchResult(this, entry->title(), desc, entry->field(TQString::fromLatin1("isbn")));
    m_entries.insert(r->uid, entry);
    emit signalResultFound(r);
  }
  stop();
}

Tellico::Data::EntryPtr SRUFetcher::fetchEntry(uint uid_) {
  return m_entries[uid_];
}

void SRUFetcher::updateEntry(Data::EntryPtr entry_) {
//  myDebug() << "SRUFetcher::updateEntry() - " << source() << ": " << entry_->title() << endl;
  TQString isbn = entry_->field(TQString::fromLatin1("isbn"));
  if(!isbn.isEmpty()) {
    search(Fetch::ISBN, isbn);
    return;
  }

  TQString lccn = entry_->field(TQString::fromLatin1("lccn"));
  if(!lccn.isEmpty()) {
    search(Fetch::LCCN, lccn);
    return;
  }

  // optimistically try searching for title and rely on Collection::sameEntry() to figure things out
  TQString t = entry_->field(TQString::fromLatin1("title"));
  if(!t.isEmpty()) {
    search(Fetch::Title, t);
    return;
  }

  myDebug() << "SRUFetcher::updateEntry() - insufficient info to search" << endl;
  emit signalDone(this); // always need to emit this if not continuing with the search
}

bool SRUFetcher::initMARCXMLHandler() {
  if(m_MARCXMLHandler) {
    return true;
  }

  TQString xsltfile = locate("appdata", TQString::fromLatin1("MARC21slim2MODS3.xsl"));
  if(xsltfile.isEmpty()) {
    kdWarning() << "SRUFetcher::initHandlers() - can not locate MARC21slim2MODS3.xsl." << endl;
    return false;
  }

  KURL u;
  u.setPath(xsltfile);

  m_MARCXMLHandler = new XSLTHandler(u);
  if(!m_MARCXMLHandler->isValid()) {
    kdWarning() << "SRUFetcher::initHandlers() - error in MARC21slim2MODS3.xsl." << endl;
    delete m_MARCXMLHandler;
    m_MARCXMLHandler = 0;
    return false;
  }
  return true;
}

bool SRUFetcher::initMODSHandler() {
  if(m_MODSHandler) {
    return true;
  }

  TQString xsltfile = locate("appdata", TQString::fromLatin1("mods2tellico.xsl"));
  if(xsltfile.isEmpty()) {
    kdWarning() << "SRUFetcher::initHandlers() - can not locate mods2tellico.xsl." << endl;
    return false;
  }

  KURL u;
  u.setPath(xsltfile);

  m_MODSHandler = new XSLTHandler(u);
  if(!m_MODSHandler->isValid()) {
    kdWarning() << "SRUFetcher::initHandlers() - error in mods2tellico.xsl." << endl;
    delete m_MODSHandler;
    m_MODSHandler = 0;
    return false;
  }
  return true;
}

Tellico::Fetch::Fetcher::Ptr SRUFetcher::libraryOfCongress(TQObject* parent_) {
  return new SRUFetcher(i18n("Library of Congress (US)"), TQString::fromLatin1("z3950.loc.gov"), 7090,
                        TQString::fromLatin1("voyager"), parent_);
}

// static
Tellico::StringMap SRUFetcher::customFields() {
  StringMap map;
  map[TQString::fromLatin1("address")]  = i18n("Address");
  map[TQString::fromLatin1("abstract")] = i18n("Abstract");
  return map;
}

Tellico::Fetch::ConfigWidget* SRUFetcher::configWidget(TQWidget* parent_) const {
  return new SRUConfigWidget(parent_, this);
}

SRUConfigWidget::SRUConfigWidget(TQWidget* parent_, const SRUFetcher* fetcher_ /*=0*/)
    : ConfigWidget(parent_) {
  TQGridLayout* l = new TQGridLayout(optionsWidget(), 4, 2);
  l->setSpacing(4);
  l->setColStretch(1, 10);

  int row = -1;
  TQLabel* label = new TQLabel(i18n("Hos&t: "), optionsWidget());
  l->addWidget(label, ++row, 0);
  m_hostEdit = new GUI::LineEdit(optionsWidget());
  connect(m_hostEdit, TQT_SIGNAL(textChanged(const TQString&)), TQT_SLOT(slotSetModified()));
  connect(m_hostEdit, TQT_SIGNAL(textChanged(const TQString&)), TQT_SIGNAL(signalName(const TQString&)));
  connect(m_hostEdit, TQT_SIGNAL(textChanged(const TQString&)), TQT_SLOT(slotCheckHost()));
  l->addWidget(m_hostEdit, row, 1);
  TQString w = i18n("Enter the host name of the server.");
  TQWhatsThis::add(label, w);
  TQWhatsThis::add(m_hostEdit, w);
  label->setBuddy(m_hostEdit);

  label = new TQLabel(i18n("&Port: "), optionsWidget());
  l->addWidget(label, ++row, 0);
  m_portSpinBox = new KIntSpinBox(0, 999999, 1, SRU_DEFAULT_PORT, 10, optionsWidget());
  connect(m_portSpinBox, TQT_SIGNAL(valueChanged(int)), TQT_SLOT(slotSetModified()));
  l->addWidget(m_portSpinBox, row, 1);
  w = i18n("Enter the port number of the server. The default is %1.").tqarg(SRU_DEFAULT_PORT);
  TQWhatsThis::add(label, w);
  TQWhatsThis::add(m_portSpinBox, w);
  label->setBuddy(m_portSpinBox);

  label = new TQLabel(i18n("Path: "), optionsWidget());
  l->addWidget(label, ++row, 0);
  m_pathEdit = new GUI::LineEdit(optionsWidget());
  connect(m_pathEdit, TQT_SIGNAL(textChanged(const TQString&)), TQT_SLOT(slotSetModified()));
  l->addWidget(m_pathEdit, row, 1);
  w = i18n("Enter the path to the database used by the server.");
  TQWhatsThis::add(label, w);
  TQWhatsThis::add(m_pathEdit, w);
  label->setBuddy(m_pathEdit);

  label = new TQLabel(i18n("Format: "), optionsWidget());
  l->addWidget(label, ++row, 0);
  m_formatCombo = new GUI::ComboBox(optionsWidget());
  m_formatCombo->insertItem(TQString::fromLatin1("MODS"), TQString::fromLatin1("mods"));
  m_formatCombo->insertItem(TQString::fromLatin1("MARCXML"), TQString::fromLatin1("marcxml"));
  m_formatCombo->insertItem(TQString::fromLatin1("Dublin Core"), TQString::fromLatin1("dc"));
  connect(m_formatCombo, TQT_SIGNAL(activated(int)), TQT_SLOT(slotSetModified()));
  l->addWidget(m_formatCombo, row, 1);
  w = i18n("Enter the result format used by the server.");
  TQWhatsThis::add(label, w);
  TQWhatsThis::add(m_formatCombo, w);
  label->setBuddy(m_formatCombo);

  l->setRowStretch(++row, 1);

  // now add additional fields widget
  addFieldsWidget(SRUFetcher::customFields(), fetcher_ ? fetcher_->m_fields : TQStringList());

  if(fetcher_) {
    m_hostEdit->setText(fetcher_->m_host);
    m_portSpinBox->setValue(fetcher_->m_port);
    m_pathEdit->setText(fetcher_->m_path);
    m_formatCombo->setCurrentData(fetcher_->m_format);
  }
  KAcceleratorManager::manage(optionsWidget());
}

void SRUConfigWidget::saveConfig(KConfigGroup& config_) {
  TQString s = m_hostEdit->text().stripWhiteSpace();
  if(!s.isEmpty()) {
    config_.writeEntry("Host", s);
  }
  int port = m_portSpinBox->value();
  if(port > 0) {
    config_.writeEntry("Port", port);
  }
  s = m_pathEdit->text().stripWhiteSpace();
  if(!s.isEmpty()) {
    config_.writeEntry("Path", s);
  }
  s = m_formatCombo->currentData().toString();
  if(!s.isEmpty()) {
    config_.writeEntry("Format", s);
  }
  saveFieldsConfig(config_);
  slotSetModified(false);
}

TQString SRUConfigWidget::preferredName() const {
  TQString s = m_hostEdit->text();
  return s.isEmpty() ? SRUFetcher::defaultName() : s;
}

void SRUConfigWidget::slotCheckHost() {
  TQString s = m_hostEdit->text();
  // someone might be pasting a full URL, check that
  if(s.find(':') > -1 || s.find('/') > -1) {
    KURL u(s);
    if(u.isValid()) {
      m_hostEdit->setText(u.host());
      if(u.port() > 0) {
        m_portSpinBox->setValue(u.port());
      }
      if(!u.path().isEmpty()) {
        m_pathEdit->setText(u.path());
      }
    }
  }
}

#include "srufetcher.moc"