kmymoney/kmymoney2/converter/webpricequote.cpp

/***************************************************************************
                          webpricequote.cpp
                             -------------------
    begin                : Thu Dec 30 2004
    copyright            : (C) 2004 by Ace Jones
    email                : Ace Jones <acejones@users.sourceforge.net>
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

// ----------------------------------------------------------------------------
// TQt Headers

#include <tqfile.h>
#include <tqregexp.h>
#include <tqtextstream.h>
#include <tqprocess.h>

// ----------------------------------------------------------------------------
// TDE Headers

#include <tdeio/netaccess.h>
#include <tdeio/scheduler.h>
#include <kurl.h>
#include <tdelocale.h>
#include <kdebug.h>
#include <tdeapplication.h>
#include <tdeconfig.h>
#include <tdeglobal.h>
#include <kstandarddirs.h>
#include <kcalendarsystem.h>
#include <tdetempfile.h>

// ----------------------------------------------------------------------------
// Project Headers

#include "../mymoney/mymoneyexception.h"
#include "mymoneyqifprofile.h"
#include "webpricequote.h"

// define static members
TQString WebPriceQuote::m_financeQuoteScriptPath;
TQStringList WebPriceQuote::m_financeQuoteSources;

TQString * WebPriceQuote::lastErrorMsg;
int WebPriceQuote::lastErrorCode = 0;

WebPriceQuote::WebPriceQuote( TQObject* _parent, const char* _name ):
  TQObject( _parent, _name )
{
  m_financeQuoteScriptPath =
      TDEGlobal::dirs()->findResource("appdata", TQString("misc/financequote.pl"));
  connect(&m_filter,TQ_SIGNAL(processExited(const TQString&)),this,TQ_SLOT(slotParseQuote(const TQString&)));
}

WebPriceQuote::~WebPriceQuote()
{
}

bool WebPriceQuote::launch( const TQString& _symbol, const TQString& _id, const TQString& _sourcename )
{
  if (_sourcename.contains("Finance::Quote"))
    return (launchFinanceQuote (_symbol, _id, _sourcename));
  else
    return (launchNative (_symbol, _id, _sourcename));
}

bool WebPriceQuote::launchNative( const TQString& _symbol, const TQString& _id, const TQString& _sourcename ) {
  bool result = true;
  m_symbol = _symbol;
  m_id = _id;

//   emit status(TQString("(Debug) symbol=%1 id=%2...").arg(_symbol,_id));

  // if we're running normally, with a UI, we can just get these the normal way,
  // from the config file
  if ( kapp )
  {
    TQString sourcename = _sourcename;
    if ( sourcename.isEmpty() )
      sourcename = "Yahoo";

    if ( quoteSources().contains(sourcename) )
      m_source = WebPriceQuoteSource(sourcename);
    else
      emit error(TQString("Source <%1> does not exist.").arg(sourcename));
  }
  // otherwise, if we have no kapp, we have no config.  so we just get them from
  // the defaults
  else
  {
    if ( _sourcename.isEmpty() )
      m_source = defaultQuoteSources()["Yahoo"];
    else
      m_source = defaultQuoteSources()[_sourcename];
  }

  KURL url;

  // if the source has room for TWO symbols..
  if ( m_source.m_url.contains("%2") )
  {
    // this is a two-symbol quote.  split the symbol into two.  valid symbol
    // characters are: 0-9, A-Z and the dot.  anything else is a separator
    TQRegExp splitrx("([0-9a-z\\.]+)[^a-z0-9]+([0-9a-z\\.]+)",false /*case sensitive*/);

    // if we've truly found 2 symbols delimited this way...
    if ( splitrx.search(m_symbol) != -1 )
      url = KURL::fromPathOrURL(m_source.m_url.arg(splitrx.cap(1),splitrx.cap(2)));
    else
      kdDebug(2) << "WebPriceQuote::launch() did not find 2 symbols" << endl;
  }
  else
    // a regular one-symbol quote
    url = KURL::fromPathOrURL(m_source.m_url.arg(m_symbol));

  // If we're running a non-interactive session (with no UI), we can't
  // use TDEIO::NetAccess, so we have to get our web data the old-fashioned
  // way... with 'wget'.
  //
  // Note that a 'non-interactive' session right now means only the test
  // cases.  Although in the future if KMM gains a non-UI mode, this would
  // still be useful
  if ( ! kapp && ! url.isLocalFile() )
    url = KURL::fromPathOrURL("/usr/bin/wget -O - " + url.prettyURL());

  if ( url.isLocalFile() )
  {
    emit status(TQString("Executing %1...").arg(url.path()));

    m_filter.clearArguments();
    m_filter << TQStringList::split(" ",url.path());
    m_filter.setSymbol(m_symbol);

    // if we're running non-interactive, we'll need to block.
    // otherwise, just let us know when it's done.
    TDEProcess::RunMode mode = TDEProcess::NotifyOnExit;
    if ( ! kapp )
      mode = TDEProcess::Block;

    if(m_filter.start(mode, TDEProcess::All))
    {
      result = true;
      m_filter.resume();
    }
    else
    {
      emit error(TQString("Unable to launch: %1").arg(url.path()));
      slotParseQuote(TQString());
    }
  }
  else
  {
    emit status(TQString("Fetching URL %1...").arg(url.prettyURL()));

    TQString tmpFile;
    if( download( url, tmpFile, NULL ) )
    {
      kdDebug(2) << "Downloaded " << tmpFile << endl;
      TQFile f(tmpFile);
      if ( f.open( IO_ReadOnly ) )
      {
        result = true;
        TQString quote = TQTextStream(&f).read();
        f.close();
        slotParseQuote(quote);
      }
      else
      {
        slotParseQuote(TQString());
      }
      removeTempFile( tmpFile );
    }
    else
    {
      emit error(TDEIO::NetAccess::lastErrorString());
      slotParseQuote(TQString());
    }
  }
  return result;
}

void WebPriceQuote::removeTempFile(const TQString& tmpFile)
{
  if(tmpFile == m_tmpFile) {
    unlink(tmpFile.local8Bit());
    m_tmpFile = TQString();
  }
}

bool WebPriceQuote::download(const KURL& u, TQString & target, TQWidget* window)
{
  m_tmpFile = TQString();

  // the following code taken and adapted from TDEIO::NetAccess::download()
  if (target.isEmpty())
  {
    KTempFile tmpFile;
    target = tmpFile.name();
    m_tmpFile = target;
  }

  KURL dest;
  dest.setPath( target );


  // the following code taken and adapted from TDEIO::NetAccess::filecopyInternal()
  bJobOK = true; // success unless further error occurs

  TDEIO::Scheduler::checkSlaveOnHold(true);
  TDEIO::Job * job = TDEIO::file_copy( u, dest, -1, true, false, false );
  job->setWindow (window);
  job->addMetaData("cache", "reload");  // bypass cache
  connect( job, TQ_SIGNAL( result (TDEIO::Job *) ),
           this, TQ_SLOT( slotResult (TDEIO::Job *) ) );

  enter_loop();
  return bJobOK;

}

// The following parts are copied and adjusted from TDEIO::NetAccess

// If a troll sees this, he kills me
void tqt_enter_modal( TQWidget *widget );
void tqt_leave_modal( TQWidget *widget );

void WebPriceQuote::enter_loop(void)
{
  TQWidget dummy(0,0,WType_Dialog | WShowModal);
  dummy.setFocusPolicy( TQWidget::NoFocus );
  tqt_enter_modal(&dummy);
  tqApp->enter_loop();
  tqt_leave_modal(&dummy);
}

void WebPriceQuote::slotResult( TDEIO::Job * job )
{
  lastErrorCode = job->error();
  bJobOK = !job->error();
  if ( !bJobOK )
  {
    if ( !lastErrorMsg )
      lastErrorMsg = new TQString;
    *lastErrorMsg = job->errorString();
  }

  tqApp->exit_loop();
}
// The above parts are copied and adjusted from TDEIO::NetAccess

bool WebPriceQuote::launchFinanceQuote ( const TQString& _symbol, const TQString& _id,
                               const TQString& _sourcename ) {
  bool result = true;
  m_symbol = _symbol;
  m_id = _id;
  TQString FTQSource = _sourcename.section (" ", 1);
  m_source = WebPriceQuoteSource (_sourcename, m_financeQuoteScriptPath,
                                  "\"([^,\"]*)\",.*",  // symbol regexp
                                  "[^,]*,[^,]*,\"([^\"]*)\"", // price regexp
                                  "[^,]*,([^,]*),.*", // date regexp
                                  "%y-%m-%d"); // date format

  //emit status(TQString("(Debug) symbol=%1 id=%2...").arg(_symbol,_id));


  m_filter.clearArguments();
  m_filter << "perl" << m_financeQuoteScriptPath << FTQSource << TDEProcess::quote(_symbol);
  m_filter.setUseShell(true);
  m_filter.setSymbol(m_symbol);
  emit status(TQString("Executing %1 %2 %3...").arg(m_financeQuoteScriptPath).arg(FTQSource).arg(_symbol));

    // if we're running non-interactive, we'll need to block.
    // otherwise, just let us know when it's done.
  TDEProcess::RunMode mode = TDEProcess::NotifyOnExit;
  if ( ! kapp )
    mode = TDEProcess::Block;

  if(m_filter.start(mode, TDEProcess::All))
  {
    result = true;
    m_filter.resume();
  }
  else
  {
    emit error(TQString("Unable to launch: %1").arg(m_financeQuoteScriptPath));
    slotParseQuote(TQString());
  }

  return result;
}

void WebPriceQuote::slotParseQuote(const TQString& _quotedata)
{
  TQString quotedata = _quotedata;
  bool gotprice = false;
  bool gotdate = false;

//    kdDebug(2) << "WebPriceQuote::slotParseQuote( " << _quotedata << " ) " << endl;

  if ( ! quotedata.isEmpty() )
  {
    if(!m_source.m_skipStripping) {
      //
      // First, remove extranous non-data elements
      //

      // HTML tags
      quotedata.remove(TQRegExp("<[^>]*>"));

      // &...;'s
      quotedata.replace(TQRegExp("&\\w+;")," ");

      // Extra white space
      quotedata = quotedata.simplifyWhiteSpace();
    }

#if KMM_DEBUG
    // Enable to get a look at the data coming back from the source after it's stripped
    TQFile file("stripped.txt");
    if ( file.open( IO_WriteOnly ) )
    {
      TQTextStream( &file ) << quotedata;
      file.close();
    }
#endif

    TQRegExp symbolRegExp(m_source.m_sym);
    TQRegExp dateRegExp(m_source.m_date);
    TQRegExp priceRegExp(m_source.m_price);

    if( symbolRegExp.search(quotedata) > -1)
      emit status(i18n("Symbol found: %1").arg(symbolRegExp.cap(1)));

    if(priceRegExp.search(quotedata)> -1)
    {
      gotprice = true;

      // Deal with european quotes that come back as X.XXX,XX or XX,XXX
      //
      // We will make the assumption that ALL prices have a decimal separator.
      // So "1,000" always means 1.0, not 1000.0.
      //
      // Remove all non-digits from the price string except the last one, and
      // set the last one to a period.
      TQString pricestr = priceRegExp.cap(1);

      int pos = pricestr.findRev(TQRegExp("\\D"));
      if ( pos > 0 )
      {
        pricestr[pos] = '.';
        pos = pricestr.findRev(TQRegExp("\\D"),pos-1);
      }
      while ( pos > 0 )
      {
        pricestr.remove(pos,1);
        pos = pricestr.findRev(TQRegExp("\\D"),pos);
      }

      m_price = pricestr.toDouble();
      emit status(i18n("Price found: %1 (%2)").arg(pricestr).arg(m_price));
    }

    if(dateRegExp.search(quotedata) > -1)
    {
      TQString datestr = dateRegExp.cap(1);

      MyMoneyDateFormat dateparse(m_source.m_dateformat);
      try
      {
        m_date = dateparse.convertString( datestr,false /*strict*/ );
        gotdate = true;
        emit status(i18n("Date found: %1").arg(m_date.toString()));;
      }
      catch (MyMoneyException* e)
      {
        // emit error(i18n("Unable to parse date %1 using format %2: %3").arg(datestr,dateparse.format(),e->what()));
        m_date = TQDate::currentDate();
        gotdate = true;
        delete e;
      }
    }

    if ( gotprice && gotdate )
    {
      emit quote( m_id, m_symbol, m_date, m_price );
    }
    else
    {
      emit error(i18n("Unable to update price for %1").arg(m_symbol));
      emit failed( m_id, m_symbol );
    }
  }
  else
  {
    emit error(i18n("Unable to update price for %1").arg(m_symbol));
    emit failed( m_id, m_symbol );
  }
}

TQMap<TQString,WebPriceQuoteSource> WebPriceQuote::defaultQuoteSources(void)
{
  TQMap<TQString,WebPriceQuoteSource> result;

  result["Yahoo"] = WebPriceQuoteSource("Yahoo",
    "http://finance.yahoo.com/d/quotes.csv?s=%1&f=sl1d1",
    "\"([^,\"]*)\",.*",  // symbolregexp
    "[^,]*,([^,]*),.*", // priceregexp
    "[^,]*,[^,]*,\"([^\"]*)\"", // dateregexp
    "%m %d %y" // dateformat
  );

  result["Yahoo Currency"] = WebPriceQuoteSource("Yahoo Currency",
    "http://finance.yahoo.com/d/quotes.csv?s=%1%2=X&f=sl1d1",
    "\"([^,\"]*)\",.*",  // symbolregexp
    "[^,]*,([^,]*),.*", // priceregexp
    "[^,]*,[^,]*,\"([^\"]*)\"", // dateregexp
    "%m %d %y" // dateformat
  );

  // 2009-08-20 Yahoo UK has no quotes and has comma separators
  // sl1d1 format for Yahoo UK doesn't seem to give a date ever
  // sl1d3 gives US locale time (9:99pm) and date (mm/dd/yyyy)
  result["Yahoo UK"] = WebPriceQuoteSource("Yahoo UK",
    "http://uk.finance.yahoo.com/d/quotes.csv?s=%1&f=sl1d3",
    "^([^,]*),.*",  // symbolregexp
    "^[^,]*,([^,]*),.*", // priceregexp
    "^[^,]*,[^,]*,(.*)", // dateregexp
    "%m/%d/%y" // dateformat
  );

  // sl1d1 format for Yahoo France doesn't seem to give a date ever
  // sl1d3 gives us time (99h99) and date
  result["Yahoo France"] = WebPriceQuoteSource("Yahoo France",
    "http://fr.finance.yahoo.com/d/quotes.csv?s=%1&f=sl1d3",
    "([^;]*).*",             // symbolregexp
    "[^;]*.([^;]*),*",       // priceregexp
    "[^;]*.[^;]*...h...([^;]*)", // dateregexp
    "%d/%m/%y"               // dateformat
  );

  result["Globe & Mail"] = WebPriceQuoteSource("Globe & Mail",
    "http://globefunddb.theglobeandmail.com/gishome/plsql/gis.price_history?pi_fund_id=%1",
    TQString(),  // symbolregexp
    "Reinvestment Price \\w+ \\d+, \\d+ (\\d+\\.\\d+)", // priceregexp
    "Reinvestment Price (\\w+ \\d+, \\d+)", // dateregexp
    "%m %d %y" // dateformat
  );

  result["MSN.CA"] = WebPriceQuoteSource("MSN.CA",
    "http://ca.moneycentral.msn.com/investor/quotes/quotes.asp?symbol=%1",
    TQString(),  // symbolregexp
    "Net Asset Value (\\d+\\.\\d+)", // priceregexp
    "NAV update (\\d+\\D+\\d+\\D+\\d+)", // dateregexp
    "%d %m %y" // dateformat
  );
  // Finanztreff (replaces VWD.DE) and boerseonline supplied by Micahel Zimmerman
  result["Finanztreff"] = WebPriceQuoteSource("Finanztreff",
    "http://finanztreff.de/kurse_einzelkurs_detail.htn?u=100&i=%1",
    TQString(),  // symbolregexp
    "([0-9]+,\\d+).+Gattung:Fonds", // priceregexp
    "\\).(\\d+\\D+\\d+\\D+\\d+)", // dateregexp (doesn't work; date in chart
    "%d.%m.%y" // dateformat
  );

  result["boerseonline"] = WebPriceQuoteSource("boerseonline",
    "http://www.boerse-online.de/tools/boerse/einzelkurs_kurse.htm?&s=%1",
    TQString(),  // symbolregexp
    "Akt\\. Kurs.(\\d+,\\d\\d)", // priceregexp
    "Datum.(\\d+\\.\\d+\\.\\d+)", // dateregexp (doesn't work; date in chart
    "%d.%m.%y" // dateformat
  );

  // The following two price sources were contributed by
  // Marc Zahnlecker <tf2k@users.sourceforge.net>

  result["Wallstreet-Online.DE (Default)"] = WebPriceQuoteSource("Wallstreet-Online.DE (Default)",
    "http://www.wallstreet-online.de/si/?k=%1&spid=ws",
    "Symbol:(\\w+)",  // symbolregexp
    "Letzter Kurs: ([0-9.]+,\\d+)", // priceregexp
    ", (\\d+\\D+\\d+\\D+\\d+)", // dateregexp
    "%d %m %y" // dateformat
  );

  // This quote source provided by Peter Lord
  // The trading symbol will normally be the SEDOL (see wikipedia) but
  // the flexibility presently (1/2008) in the code will allow use of
  // the ISIN or MEXID (FT specific) codes
  result["Financial Times UK Funds"] = WebPriceQuoteSource("Financial Times UK Funds",
      "http://funds.ft.com/funds/simpleSearch.do?searchArea=%&search=%1",
      "SEDOL[\\ ]*(\\d+.\\d+)", // symbol regexp
      "\\(GBX\\)[\\ ]*([0-9,]*.\\d+)[\\ ]*", // price regexp
      "Valuation date:[\\ ]*(\\d+/\\d+/\\d+)", // date regexp
      "%d/%m/%y" // date format
  );

  // This quote source provided by Danny Scott
  result["Yahoo Canada"] = WebPriceQuoteSource("Yahoo Canada",
      "http://ca.finance.yahoo.com/q?s=%1",
      "%1", // symbol regexp
      "Last Trade: (\\d+\\.\\d+)", // price regexp
      "day, (.\\D+\\d+\\D+\\d+)", // date regexp
      "%m %d %y" // date format
  );

  // (tf2k) The "mpid" is I think the market place id. In this case five
  // stands for Hamburg.
  //
  // Here the id for several market places: 2 Frankfurt, 3 Berlin, 4
  // Düsseldorf, 5 Hamburg, 6 München/Munich, 7 Hannover, 9 Stuttgart, 10
  // Xetra, 32 NASDAQ, 36 NYSE

  result["Wallstreet-Online.DE (Hamburg)"] = WebPriceQuoteSource("Wallstreet-Online.DE (Hamburg)",
    "http://fonds.wallstreet-online.de/si/?k=%1&spid=ws&mpid=5",
    "Symbol:(\\w+)",  // symbolregexp
    "Fonds \\(EUR\\) ([0-9.]+,\\d+)", // priceregexp
    ", (\\d+\\D+\\d+\\D+\\d+)", // dateregexp
    "%d %m %y" // dateformat
  );

  // The following price quote was contributed by
  // Piotr Adacha <piotr.adacha@googlemail.com>

  // I would like to post new Online Query Settings for KMyMoney. This set is
  // suitable to query stooq.com service, providing quotes for stocks, futures,
  // mutual funds and other financial instruments from Polish Gielda Papierow
  // Wartosciowych (GPW). Unfortunately, none of well-known international
  // services provide quotes for this market (biggest one in central and eastern
  // Europe), thus, I think it could be helpful for Polish users of KMyMoney (and
  // I am one of them for almost a year).

  result["Gielda Papierow Wartosciowych (GPW)"] = WebPriceQuoteSource("Gielda Papierow Wartosciowych (GPW)",
    "http://stooq.com/q/?s=%1",
    TQString(),                   // symbol regexp
    "Kurs.*(\\d+\\.\\d+).*Data",    // price regexp
    "(\\d{4,4}-\\d{2,2}-\\d{2,2})", // date regexp
    "%y %m %d"                   // date format
  );

  // The following price quote is for getting prices of different funds
  // at OMX Baltic market.
  result["OMX Baltic funds"] = WebPriceQuoteSource("OMX Baltic funds",
      "http://www.baltic.omxgroup.com/market/?pg=nontradeddetails&currency=0&instrument=%1",
      TQString(),  // symbolregexp
      "NAV (\\d+,\\d+)",  // priceregexp
      "Kpv (\\d+.\\d+.\\d+)",  // dateregexp
      "%d.%m.%y"   // dateformat
  );

  // The following price quote was contributed by
  // Peter Hargreaves <pete.h@pdh-online.info>
  // The original posting can be found here:
  // http://sourceforge.net/mailarchive/message.php?msg_name=200806060854.11682.pete.h%40pdh-online.info

  // I have PEP and ISA accounts which I invest in Funds with Barclays
  // Stockbrokers. They give me Fund data via Financial Express:
  //
  // https://webfund6.financialexpress.net/Clients/Barclays/default.aspx
  //
  // A typical Fund Factsheet is:
  //
  // https://webfund6.financialexpress.net/Clients/Barclays/search_factsheet_summary.aspx?code=0585239
  //
  // On the Factsheet to identify the fund you can see ISIN Code GB0005852396.
  // In the url, this code is shortened by loosing the first four and last
  // characters.
  //
  // Update:
  //
  // Nick Elliot has contributed a modified regular expression to cope with values presented
  // in pounds as well as those presented in pence. The source can be found here:
  // http://forum.kde.org/update-stock-and-currency-prices-t-32049.html

  result["Financial Express"] = WebPriceQuoteSource("Financial Express",
       "https://webfund6.financialexpress.net/Clients/Barclays/search_factsheet_summary.aspx?code=%1",
       "ISIN Code[^G]*(GB..........).*",  // symbolregexp
       "Current Market Information[^0-9]*([0-9,\\.]+).*", // priceregexp
       "Price Date[^0-9]*(../../....).*", // dateregexp
       "%d/%m/%y"                         // dateformat
  );

  return result;
}

TQStringList WebPriceQuote::quoteSources (const _quoteSystemE _system) {
  if (_system == Native)
    return (quoteSourcesNative());
  else
    return (quoteSourcesFinanceQuote());
}

TQStringList WebPriceQuote::quoteSourcesNative()
{
  TDEConfig *tdeconfig = TDEGlobal::config();
  TQStringList groups = tdeconfig->groupList();

  TQStringList::Iterator it;
  TQRegExp onlineQuoteSource(TQString("^Online-Quote-Source-(.*)$"));

  // get rid of all 'non online quote source' entries
  for(it = groups.begin(); it != groups.end(); it = groups.remove(it)) {
    if(onlineQuoteSource.search(*it) >= 0) {
      // Insert the name part
      groups.insert(it, onlineQuoteSource.cap(1));
    }
  }

  // if the user has the OLD quote source defined, now is the
  // time to remove that entry and convert it to the new system.
  if ( ! groups.count() && tdeconfig->hasGroup("Online Quotes Options") )
  {
    tdeconfig->setGroup("Online Quotes Options");
    TQString url(tdeconfig->readEntry("URL","http://finance.yahoo.com/d/quotes.csv?s=%1&f=sl1d1"));
    TQString symbolRegExp(tdeconfig->readEntry("SymbolRegex","\"([^,\"]*)\",.*"));
    TQString priceRegExp(tdeconfig->readEntry("PriceRegex","[^,]*,([^,]*),.*"));
    TQString dateRegExp(tdeconfig->readEntry("DateRegex","[^,]*,[^,]*,\"([^\"]*)\""));
    tdeconfig->deleteGroup("Online Quotes Options");

    groups += "Old Source";
    tdeconfig->setGroup(TQString("Online-Quote-Source-%1").arg("Old Source"));
    tdeconfig->writeEntry("URL", url);
    tdeconfig->writeEntry("SymbolRegex", symbolRegExp);
    tdeconfig->writeEntry("PriceRegex",priceRegExp);
    tdeconfig->writeEntry("DateRegex", dateRegExp);
    tdeconfig->writeEntry("DateFormatRegex", "%m %d %y");
    tdeconfig->sync();
  }

  // Set up each of the default sources.  These are done piecemeal so that
  // when we add a new source, it's automatically picked up.
  TQMap<TQString,WebPriceQuoteSource> defaults = defaultQuoteSources();
  TQMap<TQString,WebPriceQuoteSource>::const_iterator it_source = defaults.begin();
  while ( it_source != defaults.end() )
  {
    if ( ! groups.contains( (*it_source).m_name ) )
    {
      groups += (*it_source).m_name;
      (*it_source).write();
      tdeconfig->sync();
    }
    ++it_source;
  }

  return groups;
}

TQStringList WebPriceQuote::quoteSourcesFinanceQuote()
{
  if (m_financeQuoteSources.empty()) { // run the process one time only
    FinanceQuoteProcess getList;
    m_financeQuoteScriptPath =
        TDEGlobal::dirs()->findResource("appdata", TQString("misc/financequote.pl"));
    getList.launch( m_financeQuoteScriptPath );
    while (!getList.isFinished()) {
      tqApp->processEvents();
    }
    m_financeQuoteSources = getList.getSourceList();
  }
  return (m_financeQuoteSources);
}

//
// Helper class to load/save an individual source
//

WebPriceQuoteSource::WebPriceQuoteSource(const TQString& name, const TQString& url, const TQString& sym, const TQString& price, const TQString& date, const TQString& dateformat):
  m_name(name),
  m_url(url),
  m_sym(sym),
  m_price(price),
  m_date(date),
  m_dateformat(dateformat)
{
}

WebPriceQuoteSource::WebPriceQuoteSource(const TQString& name)
{
  m_name = name;
  TDEConfig *tdeconfig = TDEGlobal::config();
  tdeconfig->setGroup(TQString("Online-Quote-Source-%1").arg(m_name));
  m_sym = tdeconfig->readEntry("SymbolRegex");
  m_date = tdeconfig->readEntry("DateRegex");
  m_dateformat = tdeconfig->readEntry("DateFormatRegex","%m %d %y");
  m_price = tdeconfig->readEntry("PriceRegex");
  m_url = tdeconfig->readEntry("URL");
  m_skipStripping = tdeconfig->readBoolEntry("SkipStripping", false);
}

void WebPriceQuoteSource::write(void) const
{
  TDEConfig *tdeconfig = TDEGlobal::config();
  tdeconfig->setGroup(TQString("Online-Quote-Source-%1").arg(m_name));
  tdeconfig->writeEntry("URL", m_url);
  tdeconfig->writeEntry("PriceRegex", m_price);
  tdeconfig->writeEntry("DateRegex", m_date);
  tdeconfig->writeEntry("DateFormatRegex", m_dateformat);
  tdeconfig->writeEntry("SymbolRegex", m_sym);
  if(m_skipStripping)
    tdeconfig->writeEntry("SkipStripping", m_skipStripping);
  else
    tdeconfig->deleteEntry("SkipStripping");
}

void WebPriceQuoteSource::rename(const TQString& name)
{
  remove();
  m_name = name;
  write();
}

void WebPriceQuoteSource::remove(void) const
{
  TDEConfig *tdeconfig = TDEGlobal::config();
  tdeconfig->deleteGroup(TQString("Online-Quote-Source-%1").arg(m_name));
}

//
// Helper class to babysit the TDEProcess used for running the local script in that case
//

WebPriceQuoteProcess::WebPriceQuoteProcess(void)
{
  connect(this, TQ_SIGNAL(receivedStdout(TDEProcess*, char*, int)), this, TQ_SLOT(slotReceivedDataFromFilter(TDEProcess*, char*, int)));
  connect(this, TQ_SIGNAL(processExited(TDEProcess*)), this, TQ_SLOT(slotProcessExited(TDEProcess*)));
}

void WebPriceQuoteProcess::slotReceivedDataFromFilter(TDEProcess* /*_process*/, char* _pcbuffer, int _nbufferlen)
{
  TQByteArray data;
  data.duplicate(_pcbuffer, _nbufferlen);

//   kdDebug(2) << "WebPriceQuoteProcess::slotReceivedDataFromFilter(): " << TQString(data) << endl;
  m_string += TQString(data);
}

void WebPriceQuoteProcess::slotProcessExited(TDEProcess*)
{
//   kdDebug(2) << "WebPriceQuoteProcess::slotProcessExited()" << endl;
  emit processExited(m_string);
  m_string.truncate(0);
}

//
// Helper class to babysit the TDEProcess used for running the Finance Quote sources script
//

FinanceQuoteProcess::FinanceQuoteProcess(void)
{
  m_isDone = false;
  m_string = "";
  m_fqNames["aex"] = "AEX";
  m_fqNames["aex_futures"] = "AEX Futures";
  m_fqNames["aex_options"] = "AEX Options";
  m_fqNames["amfiindia"] = "AMFI India";
  m_fqNames["asegr"] = "ASE";
  m_fqNames["asia"] = "Asia (Yahoo, ...)";
  m_fqNames["asx"] = "ASX";
  m_fqNames["australia"] = "Australia (ASX, Yahoo, ...)";
  m_fqNames["bmonesbittburns"] = "BMO NesbittBurns";
  m_fqNames["brasil"] = "Brasil (Yahoo, ...)";
  m_fqNames["canada"] = "Canada (Yahoo, ...)";
  m_fqNames["canadamutual"] = "Canada Mutual (Fund Library, ...)";
  m_fqNames["deka"] = "Deka Investments";
  m_fqNames["dutch"] = "Dutch (AEX, ...)";
  m_fqNames["dwsfunds"] = "DWS";
  m_fqNames["europe"] = "Europe (Yahoo, ...)";
  m_fqNames["fidelity"] = "Fidelity (Fidelity, ...)";
  m_fqNames["fidelity_direct"] = "Fidelity Direct";
  m_fqNames["financecanada"] = "Finance Canada";
  m_fqNames["ftportfolios"] = "First Trust (First Trust, ...)";
  m_fqNames["ftportfolios_direct"] = "First Trust Portfolios";
  m_fqNames["fundlibrary"] = "Fund Library";
  m_fqNames["greece"] = "Greece (ASE, ...)";
  m_fqNames["indiamutual"] = "India Mutual (AMFI, ...)";
  m_fqNames["maninv"] = "Man Investments";
  m_fqNames["fool"] = "Motley Fool";
  m_fqNames["nasdaq"] = "Nasdaq (Yahoo, ...)";
  m_fqNames["nz"] = "New Zealand (Yahoo, ...)";
  m_fqNames["nyse"] = "NYSE (Yahoo, ...)";
  m_fqNames["nzx"] = "NZX";
  m_fqNames["platinum"] = "Platinum Asset Management";
  m_fqNames["seb_funds"] = "SEB";
  m_fqNames["sharenet"] = "Sharenet";
  m_fqNames["za"] = "South Africa (Sharenet, ...)";
  m_fqNames["troweprice_direct"] = "T. Rowe Price";
  m_fqNames["troweprice"] = "T. Rowe Price";
  m_fqNames["tdefunds"] = "TD Efunds";
  m_fqNames["tdwaterhouse"] = "TD Waterhouse Canada";
  m_fqNames["tiaacref"] = "TIAA-CREF";
  m_fqNames["trustnet"] = "Trustnet";
  m_fqNames["uk_unit_trusts"] = "U.K. Unit Trusts";
  m_fqNames["unionfunds"] = "Union Investments";
  m_fqNames["tsp"] = "US Govt. Thrift Savings Plan";
  m_fqNames["usfedbonds"] = "US Treasury Bonds";
  m_fqNames["usa"] = "USA (Yahoo, Fool ...)";
  m_fqNames["vanguard"] = "Vanguard";
  m_fqNames["vwd"] = "VWD";
  m_fqNames["yahoo"] = "Yahoo";
  m_fqNames["yahoo_asia"] = "Yahoo Asia";
  m_fqNames["yahoo_australia"] = "Yahoo Australia";
  m_fqNames["yahoo_brasil"] = "Yahoo Brasil";
  m_fqNames["yahoo_europe"] = "Yahoo Europe";
  m_fqNames["yahoo_nz"] = "Yahoo New Zealand";
  m_fqNames["zifunds"] = "Zuerich Investments";
  connect(this, TQ_SIGNAL(receivedStdout(TDEProcess*, char*, int)), this, TQ_SLOT(slotReceivedDataFromFilter(TDEProcess*, char*, int)));
  connect(this, TQ_SIGNAL(processExited(TDEProcess*)), this, TQ_SLOT(slotProcessExited(TDEProcess*)));
}

void FinanceQuoteProcess::slotReceivedDataFromFilter(TDEProcess* /*_process*/, char* _pcbuffer, int _nbufferlen)
{
  TQByteArray data;
  data.duplicate(_pcbuffer, _nbufferlen);

//   kdDebug(2) << "WebPriceQuoteProcess::slotReceivedDataFromFilter(): " << TQString(data) << endl;
  m_string += TQString(data);
}

void FinanceQuoteProcess::slotProcessExited(TDEProcess*)
{
//   kdDebug(2) << "WebPriceQuoteProcess::slotProcessExited()" << endl;
  m_isDone = true;
}

void FinanceQuoteProcess::launch (const TQString& scriptPath) {
  clearArguments();
  arguments.append(TQCString("perl"));
  arguments.append (scriptPath.local8Bit());
  arguments.append (TQCString("-l"));
  if (!start(TDEProcess::NotifyOnExit, TDEProcess::Stdout)) tqFatal ("Unable to start FQ script");
  return;
}

TQStringList FinanceQuoteProcess::getSourceList() {
  TQStringList raw = TQStringList::split(0x0A, m_string);
  TQStringList sources;
  TQStringList::iterator it;
  for (it = raw.begin(); it != raw.end(); ++it) {
    if (m_fqNames[*it].isEmpty()) sources.append(*it);
    else sources.append(m_fqNames[*it]);
  }
  sources.sort();
  return (sources);
}

const TQString FinanceQuoteProcess::crypticName(const TQString& niceName) {
  TQString ret (niceName);
  fqNameMap::iterator it;
  for (it = m_fqNames.begin(); it != m_fqNames.end(); ++it) {
    if (niceName == it.data()) {
      ret = it.key();
      break;
    }
  }
  return (ret);
}

const TQString FinanceQuoteProcess::niceName(const TQString& crypticName) {
  TQString ret (m_fqNames[crypticName]);
  if (ret.isEmpty()) ret = crypticName;
  return (ret);
}
//
// Universal date converter
//

// In 'strict' mode, this is designed to be compatable with the QIF profile date
// converter.  However, that converter deals with the concept of an apostrophe
// format in a way I don't understand.  So for the moment, they are 99%
// compatable, waiting on that issue. (acejones)

TQDate MyMoneyDateFormat::convertString(const TQString& _in, bool _strict, unsigned _centurymidpoint) const
{
  //
  // Break date format string into component parts
  //

  TQRegExp formatrex("%([mdy]+)(\\W+)%([mdy]+)(\\W+)%([mdy]+)",false /* case sensitive */);
  if ( formatrex.search(m_format) == -1 )
  {
    throw new MYMONEYEXCEPTION("Invalid format string");
  }

  TQStringList formatParts;
  formatParts += formatrex.cap(1);
  formatParts += formatrex.cap(3);
  formatParts += formatrex.cap(5);

  TQStringList formatDelimiters;
  formatDelimiters += formatrex.cap(2);
  formatDelimiters += formatrex.cap(4);

  //
  // Break input string up into component parts,
  // using the delimiters found in the format string
  //

  TQRegExp inputrex;
  inputrex.setCaseSensitive(false);

  // strict mode means we must enforce the delimiters as specified in the
  // format.  non-strict allows any delimiters
  if ( _strict )
    inputrex.setPattern(TQString("(\\w+)%1(\\w+)%2(\\w+)").arg(formatDelimiters[0],formatDelimiters[1]));
  else
    inputrex.setPattern("(\\w+)\\W+(\\w+)\\W+(\\w+)");

  if ( inputrex.search(_in) == -1 )
  {
    throw new MYMONEYEXCEPTION("Invalid input string");
  }

  TQStringList scannedParts;
  scannedParts += inputrex.cap(1).lower();
  scannedParts += inputrex.cap(2).lower();
  scannedParts += inputrex.cap(3).lower();

  //
  // Convert the scanned parts into actual date components
  //

  unsigned day = 0, month = 0, year = 0;
  bool ok;
  TQRegExp digitrex("(\\d+)");
  TQStringList::const_iterator it_scanned = scannedParts.begin();
  TQStringList::const_iterator it_format = formatParts.begin();
  while ( it_scanned != scannedParts.end() )
  {
    switch ( (*it_format)[0] )
    {
    case 'd':
      // remove any extraneous non-digits (e.g. read "3rd" as 3)
      ok = false;
      if ( digitrex.search(*it_scanned) != -1 )
        day = digitrex.cap(1).toUInt(&ok);
      if ( !ok || day > 31 )
        throw new MYMONEYEXCEPTION(TQString("Invalid day entry: %1").arg(*it_scanned));
      break;
    case 'm':
      month = (*it_scanned).toUInt(&ok);
      if ( !ok )
      {
        // maybe it's a textual date
        unsigned i = 1;
        while ( i <= 12 )
        {
          if(TDEGlobal::locale()->calendar()->monthName(i, 2000, true).lower() == *it_scanned
          || TDEGlobal::locale()->calendar()->monthName(i, 2000, false).lower() == *it_scanned)
            month = i;
          ++i;
        }
      }

      if ( month < 1 || month > 12 )
        throw new MYMONEYEXCEPTION(TQString("Invalid month entry: %1").arg(*it_scanned));

      break;
    case 'y':
      if ( _strict && (*it_scanned).length() != (*it_format).length())
        throw new MYMONEYEXCEPTION(TQString("Length of year (%1) does not match expected length (%2).")
                .arg(*it_scanned,*it_format));

      year = (*it_scanned).toUInt(&ok);

      if (!ok)
        throw new MYMONEYEXCEPTION(TQString("Invalid year entry: %1").arg(*it_scanned));

      //
      // 2-digit year case
      //
      // this algorithm will pick a year within +/- 50 years of the
      // centurymidpoint parameter.  i.e. if the midpoint is 2000,
      // then 0-49 will become 2000-2049, and 50-99 will become 1950-1999
      if ( year < 100 )
      {
        unsigned centuryend = _centurymidpoint + 50;
        unsigned centurybegin = _centurymidpoint - 50;

        if ( year < centuryend % 100 )
          year += 100;
        year += centurybegin - centurybegin % 100;
      }

      if ( year < 1900 )
        throw new MYMONEYEXCEPTION(TQString("Invalid year (%1)").arg(year));

      break;
    default:
      throw new MYMONEYEXCEPTION("Invalid format character");
    }

    ++it_scanned;
    ++it_format;
  }

  TQDate result(year,month,day);
  if ( ! result.isValid() )
    throw new MYMONEYEXCEPTION(TQString("Invalid date (yr%1 mo%2 dy%3)").arg(year).arg(month).arg(day));

  return result;
}

//
// Unit test helpers
//

convertertest::QuoteReceiver::QuoteReceiver(WebPriceQuote* q, TQObject* parent, const char *name) :
  TQObject(parent,name)
{
  connect(q,TQ_SIGNAL(quote(const TQString&,const TQDate&, const double&)),
    this,TQ_SLOT(slotGetQuote(const TQString&,const TQDate&, const double&)));
  connect(q,TQ_SIGNAL(status(const TQString&)),
    this,TQ_SLOT(slotStatus(const TQString&)));
  connect(q,TQ_SIGNAL(error(const TQString&)),
    this,TQ_SLOT(slotError(const TQString&)));
}

convertertest::QuoteReceiver::~QuoteReceiver()
{
}

void convertertest::QuoteReceiver::slotGetQuote(const TQString&,const TQDate& d, const double& m)
{
//   kdDebug(2) << "test::QuoteReceiver::slotGetQuote( , " << d << " , " << m.toString() << " )" << endl;

  m_price = MyMoneyMoney(m);
  m_date = d;
}
void convertertest::QuoteReceiver::slotStatus(const TQString& msg)
{
//   kdDebug(2) << "test::QuoteReceiver::slotStatus( " << msg << " )" << endl;

  m_statuses += msg;
}
void convertertest::QuoteReceiver::slotError(const TQString& msg)
{
//   kdDebug(2) << "test::QuoteReceiver::slotError( " << msg << " )" << endl;

  m_errors += msg;
}
#include "webpricequote.moc"