TDE
/
ktorrent
mirror of https://mirror.git.trinitydesktop.org/gitea/TDE/ktorrent
0
0
Fork 0
Code Issues Releases Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'd46a1fe346'
${ noResults }
ktorrent/libktorrent/torrent/multifilecache.cpp

868 lines
22 KiB
Raw Blame History

/***************************************************************************
* Copyright (C) 2005 by Joris Guisson *
* joris.guisson@gmail.com *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program; if not, write to the *
* Free Software Foundation, Inc., *
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. *
***************************************************************************/
#include <errno.h>
#include <tqdir.h>
#include <tqstringlist.h>
#include <tqfileinfo.h>
#include <klocale.h>
#include <kio/netaccess.h>
#include <util/file.h>
#include <util/fileops.h>
#include <util/functions.h>
#include <util/error.h>
#include <util/log.h>
#include "torrent.h"
#include "cache.h"
#include "multifilecache.h"
#include "globals.h"
#include "chunk.h"
#include "cachefile.h"
#include "dndfile.h"
#include "preallocationthread.h"
#include "movedatafilesjob.h"
namespace bt
{
static Uint64 FileOffset(Chunk* c,const TorrentFile & f,Uint64 chunk_size);
static Uint64 FileOffset(Uint32 cindex,const TorrentFile & f,Uint64 chunk_size);
static void DeleteEmptyDirs(const TQString & output_dir,const TQString & fpath);
MultiFileCache::MultiFileCache(Torrent& tor,const TQString & tmpdir,const TQString & datadir,bool custom_output_name) : Cache(tor, tmpdir,datadir)
{
cache_dir = tmpdir + "cache" + bt::DirSeparator();
if (datadir.length() == 0)
this->datadir = guessDataDir();
if (!custom_output_name)
output_dir = this->datadir + tor.getNameSuggestion() + bt::DirSeparator();
else
output_dir = this->datadir;
files.setAutoDelete(true);
}
MultiFileCache::~MultiFileCache()
{}
TQString MultiFileCache::guessDataDir()
{
for (Uint32 i = 0;i < tor.getNumFiles();i++)
{
TorrentFile & tf = tor.getFile(i);
if (tf.doNotDownload())
continue;
TQString p = cache_dir + tf.getPath();
TQFileInfo fi(p);
if (!fi.isSymLink())
continue;
TQString dst = fi.readLink();
TQString tmp = tor.getNameSuggestion() + bt::DirSeparator() + tf.getPath();
dst = dst.left(dst.length() - tmp.length());
if (dst.length() == 0)
continue;
if (!dst.endsWith(bt::DirSeparator()))
dst += bt::DirSeparator();
Out() << "Guessed outputdir to be " << dst << endl;
return dst;
}
return TQString();
}
TQString MultiFileCache::getOutputPath() const
{
return output_dir;
}
void MultiFileCache::close()
{
files.clear();
}
void MultiFileCache::open()
{
TQString dnd_dir = tmpdir + "dnd" + bt::DirSeparator();
// open all files
for (Uint32 i = 0;i < tor.getNumFiles();i++)
{
TorrentFile & tf = tor.getFile(i);
CacheFile* fd = 0;
DNDFile* dfd = 0;
try
{
if (!tf.doNotDownload())
{
if (files.contains(i))
files.erase(i);
fd = new CacheFile();
fd->open(cache_dir + tf.getPath(),tf.getSize());
files.insert(i,fd);
}
else
{
if (dnd_files.contains(i))
dnd_files.erase(i);
dfd = new DNDFile(dnd_dir + tf.getPath() + ".dnd");
dfd->checkIntegrity();
dnd_files.insert(i,dfd);
}
}
catch (...)
{
delete fd;
fd = 0;
delete dfd;
dfd = 0;
throw;
}
}
}
void MultiFileCache::changeTmpDir(const TQString& ndir)
{
Cache::changeTmpDir(ndir);
cache_dir = tmpdir + "cache/";
TQString dnd_dir = tmpdir + "dnd" + bt::DirSeparator();
// change paths for individual files, it should not
// be a problem to move these files when they are open
for (Uint32 i = 0;i < tor.getNumFiles();i++)
{
TorrentFile & tf = tor.getFile(i);
if (tf.doNotDownload())
{
DNDFile* dfd = dnd_files.find(i);
if (dfd)
dfd->changePath(dnd_dir + tf.getPath() + ".dnd");
}
else
{
CacheFile* fd = files.find(i);
if (fd)
fd->changePath(cache_dir + tf.getPath());
}
}
}
void MultiFileCache::changeOutputPath(const TQString & outputpath)
{
output_dir = outputpath;
if (!output_dir.endsWith(bt::DirSeparator()))
output_dir += bt::DirSeparator();
datadir = output_dir;
if (!bt::Exists(cache_dir))
MakeDir(cache_dir);
for (Uint32 i = 0;i < tor.getNumFiles();i++)
{
TorrentFile & tf = tor.getFile(i);
if (!tf.doNotDownload())
{
TQString fpath = tf.getPath();
if (bt::Exists(output_dir + fpath))
{
bt::Delete(cache_dir + fpath,true); // delete any existing symlinks
// create new one
bt::SymLink(output_dir + fpath,cache_dir + fpath,true);
}
}
}
}
TDEIO::Job* MultiFileCache::moveDataFiles(const TQString & ndir)
{
if (!bt::Exists(ndir))
bt::MakeDir(ndir);
TQString nd = ndir;
if (!nd.endsWith(bt::DirSeparator()))
nd += bt::DirSeparator();
try
{
MoveDataFilesJob* mvd = new MoveDataFilesJob();
for (Uint32 i = 0;i < tor.getNumFiles();i++)
{
TorrentFile & tf = tor.getFile(i);
if (tf.doNotDownload())
continue;
// check if every directory along the path exists, and if it doesn't
// create it
TQStringList sl = TQStringList::split(bt::DirSeparator(),nd + tf.getPath());
TQString odir = bt::DirSeparator();
for (Uint32 i = 0;i < sl.count() - 1;i++)
{
odir += sl[i] + bt::DirSeparator();
if (!bt::Exists(odir))
{
bt::MakeDir(odir);
}
}
mvd->addMove(output_dir + tf.getPath(),nd + tf.getPath());
}
mvd->startMoving();
return mvd;
}
catch (bt::Error & err)
{
throw; // rethrow error
}
return 0;
}
void MultiFileCache::moveDataFilesCompleted(TDEIO::Job* job)
{
if (!job->error())
{
for (Uint32 i = 0;i < tor.getNumFiles();i++)
{
TorrentFile & tf = tor.getFile(i);
// check for empty directories and delete them
DeleteEmptyDirs(output_dir,tf.getPath());
}
}
}
void MultiFileCache::create()
{
if (!bt::Exists(cache_dir))
MakeDir(cache_dir);
if (!bt::Exists(output_dir))
MakeDir(output_dir);
if (!bt::Exists(tmpdir + "dnd"))
bt::MakeDir(tmpdir + "dnd");
// update symlinks
for (Uint32 i = 0;i < tor.getNumFiles();i++)
{
TorrentFile & tf = tor.getFile(i);
touch(tf);
}
}
void MultiFileCache::touch(TorrentFile & tf)
{
TQString fpath = tf.getPath();
bool dnd = tf.doNotDownload();
// first split fpath by / separator
TQStringList sl = TQStringList::split(bt::DirSeparator(),fpath);
// create all necessary subdirs
TQString ctmp = cache_dir;
TQString otmp = output_dir;
TQString dtmp = tmpdir + "dnd" + bt::DirSeparator();
for (Uint32 i = 0;i < sl.count() - 1;i++)
{
otmp += sl[i];
ctmp += sl[i];
dtmp += sl[i];
// we need to make the same directory structure in the cache,
// the output_dir and the dnd directory
if (!bt::Exists(ctmp))
MakeDir(ctmp);
if (!bt::Exists(otmp))
MakeDir(otmp);
if (!bt::Exists(dtmp))
MakeDir(dtmp);
otmp += bt::DirSeparator();
ctmp += bt::DirSeparator();
dtmp += bt::DirSeparator();
}
bt::Delete(cache_dir + fpath,true); // delete any existing symlinks
// then make the file
TQString tmp = dnd ? tmpdir + "dnd" + bt::DirSeparator() : output_dir;
if (dnd)
{
// only symlink, when we open the files a default dnd file will be made if the file is corrupt or doesn't exist
bt::SymLink(tmp + fpath + ".dnd",cache_dir + fpath);
}
else
{
if (!bt::Exists(tmp + fpath))
{
bt::Touch(tmp + fpath);
}
else
{
preexisting_files = true;
tf.setPreExisting(true); // mark the file as preexisting
}
bt::SymLink(tmp + fpath,cache_dir + fpath);
}
}
void MultiFileCache::load(Chunk* c)
{
TQValueList<Uint32> tflist;
tor.calcChunkPos(c->getIndex(),tflist);
// one file is simple, just mmap it
if (tflist.count() == 1)
{
const TorrentFile & f = tor.getFile(tflist.first());
CacheFile* fd = files.find(tflist.first());
if (!fd)
return;
if (Cache::mappedModeAllowed() && mmap_failures < 3)
{
Uint64 off = FileOffset(c,f,tor.getChunkSize());
Uint8* buf = (Uint8*)fd->map(c,off,c->getSize(),CacheFile::READ);
if (buf)
{
c->setData(buf,Chunk::MMAPPED);
// only return when the mapping is OK
// if mmap fails we will just load it buffered
return;
}
else
mmap_failures++;
}
}
Uint8* data = new Uint8[c->getSize()];
Uint64 read = 0; // number of bytes read
for (Uint32 i = 0;i < tflist.count();i++)
{
const TorrentFile & f = tor.getFile(tflist[i]);
CacheFile* fd = files.find(tflist[i]);
DNDFile* dfd = dnd_files.find(tflist[i]);
// first calculate offset into file
// only the first file can have an offset
// the following files will start at the beginning
Uint64 off = 0;
if (i == 0)
off = FileOffset(c,f,tor.getChunkSize());
Uint32 to_read = 0;
// then the amount of data we can read from this file
if (tflist.count() == 1)
to_read = c->getSize();
else if (i == 0)
to_read = f.getLastChunkSize();
else if (i == tflist.count() - 1)
to_read = c->getSize() - read;
else
to_read = f.getSize();
// read part of data
if (fd)
fd->read(data + read,to_read,off);
else if (dfd)
{
Uint32 ret = 0;
if (i == 0)
ret = dfd->readLastChunk(data,read,c->getSize());
else if (i == tflist.count() - 1)
ret = dfd->readFirstChunk(data,read,c->getSize());
else
ret = dfd->readFirstChunk(data,read,c->getSize());
if (ret > 0 && ret != to_read)
Out() << "Warning : MultiFileCache::load ret != to_read" << endl;
}
read += to_read;
}
c->setData(data,Chunk::BUFFERED);
}
bool MultiFileCache::prep(Chunk* c)
{
// find out in which files a chunk lies
TQValueList<Uint32> tflist;
tor.calcChunkPos(c->getIndex(),tflist);
// Out() << "Prep " << c->getIndex() << endl;
if (tflist.count() == 1)
{
// in one so just mmap it
Uint64 off = FileOffset(c,tor.getFile(tflist.first()),tor.getChunkSize());
CacheFile* fd = files.find(tflist.first());
Uint8* buf = 0;
if (fd && Cache::mappedModeAllowed() && mmap_failures < 3)
{
buf = (Uint8*)fd->map(c,off,c->getSize(),CacheFile::RW);
if (!buf)
mmap_failures++;
}
if (!buf)
{
// if mmap fails or is not possible use buffered mode
c->allocate();
c->setStatus(Chunk::BUFFERED);
}
else
{
c->setData(buf,Chunk::MMAPPED);
}
}
else
{
// just allocate it
c->allocate();
c->setStatus(Chunk::BUFFERED);
}
return true;
}
void MultiFileCache::save(Chunk* c)
{
TQValueList<Uint32> tflist;
tor.calcChunkPos(c->getIndex(),tflist);
if (c->getStatus() == Chunk::MMAPPED)
{
// mapped chunks are easy
CacheFile* fd = files.find(tflist[0]);
if (!fd)
return;
fd->unmap(c->getData(),c->getSize());
c->clear();
c->setStatus(Chunk::ON_DISK);
return;
}
// Out() << "Writing to " << tflist.count() << " files " << endl;
Uint64 written = 0; // number of bytes written
for (Uint32 i = 0;i < tflist.count();i++)
{
const TorrentFile & f = tor.getFile(tflist[i]);
CacheFile* fd = files.find(tflist[i]);
DNDFile* dfd = dnd_files.find(tflist[i]);
// first calculate offset into file
// only the first file can have an offset
// the following files will start at the beginning
Uint64 off = 0;
Uint32 to_write = 0;
if (i == 0)
{
off = FileOffset(c,f,tor.getChunkSize());
}
// the amount of data we can write to this file
if (tflist.count() == 1)
to_write = c->getSize();
else if (i == 0)
to_write = f.getLastChunkSize();
else if (i == tflist.count() - 1)
to_write = c->getSize() - written;
else
to_write = f.getSize();
// Out() << "to_write " << to_write << endl;
// write the data
if (fd)
fd->write(c->getData() + written,to_write,off);
else if (dfd)
{
if (i == 0)
dfd->writeLastChunk(c->getData() + written,to_write);
else if (i == tflist.count() - 1)
dfd->writeFirstChunk(c->getData() + written,to_write);
else
dfd->writeFirstChunk(c->getData() + written,to_write);
}
written += to_write;
}
// set the chunk to on disk and clear it
c->clear();
c->setStatus(Chunk::ON_DISK);
}
void MultiFileCache::downloadStatusChanged(TorrentFile* tf, bool download)
{
bool dnd = !download;
TQString dnd_dir = tmpdir + "dnd" + bt::DirSeparator();
// if it is dnd and it is already in the dnd tree do nothing
if (dnd && bt::Exists(dnd_dir + tf->getPath() + ".dnd"))
return;
// if it is !dnd and it is already in the output_dir tree do nothing
if (!dnd && bt::Exists(output_dir + tf->getPath()))
return;
DNDFile* dfd = 0;
CacheFile* fd = 0;
try
{
if (dnd && bt::Exists(dnd_dir + tf->getPath()))
{
// old download, we need to convert it
// save first and last chunk of the file
saveFirstAndLastChunk(tf,dnd_dir + tf->getPath(),dnd_dir + tf->getPath() + ".dnd");
// delete symlink
bt::Delete(cache_dir + tf->getPath());
bt::Delete(dnd_dir + tf->getPath()); // delete old dnd file
// recreate it
bt::SymLink(dnd_dir + tf->getPath() + ".dnd",cache_dir + tf->getPath());
files.erase(tf->getIndex());
dfd = new DNDFile(dnd_dir + tf->getPath() + ".dnd");
dfd->checkIntegrity();
dnd_files.insert(tf->getIndex(),dfd);
}
else if (dnd)
{
// save first and last chunk of the file
if (bt::Exists(output_dir + tf->getPath()))
saveFirstAndLastChunk(tf,output_dir + tf->getPath(),dnd_dir + tf->getPath() + ".dnd");
// delete symlink
bt::Delete(cache_dir + tf->getPath());
// delete data file
bt::Delete(output_dir + tf->getPath(),true);
// recreate it
bt::SymLink(dnd_dir + tf->getPath() + ".dnd",cache_dir + tf->getPath());
files.erase(tf->getIndex());
dfd = new DNDFile(dnd_dir + tf->getPath() + ".dnd");
dfd->checkIntegrity();
dnd_files.insert(tf->getIndex(),dfd);
}
else
{
// recreate the file
recreateFile(tf,dnd_dir + tf->getPath() + ".dnd",output_dir + tf->getPath());
// delete symlink and dnd file
bt::Delete(cache_dir + tf->getPath());
bt::Delete(dnd_dir + tf->getPath() + ".dnd");
// recreate it
bt::SymLink(output_dir + tf->getPath(),cache_dir + tf->getPath());
dnd_files.erase(tf->getIndex());
fd = new CacheFile();
fd->open(output_dir + tf->getPath(),tf->getSize());
files.insert(tf->getIndex(),fd);
}
}
catch (bt::Error & err)
{
delete fd;
delete dfd;
Out() << err.toString() << endl;
}
}
void MultiFileCache::saveFirstAndLastChunk(TorrentFile* tf,const TQString & src_file,const TQString & dst_file)
{
DNDFile out(dst_file);
File fptr;
if (!fptr.open(src_file,"rb"))
throw Error(i18n("Cannot open file %1 : %2").arg(src_file).arg(fptr.errorString()));
Uint32 cs = 0;
if (tf->getFirstChunk() == tor.getNumChunks() - 1)
{
cs = tor.getFileLength() % tor.getChunkSize();
if (cs == 0)
cs = tor.getChunkSize();
}
else
cs = tor.getChunkSize();
Uint8* tmp = new Uint8[tor.getChunkSize()];
try
{
fptr.read(tmp,cs - tf->getFirstChunkOffset());
out.writeFirstChunk(tmp,cs - tf->getFirstChunkOffset());
if (tf->getFirstChunk() != tf->getLastChunk())
{
Uint64 off = FileOffset(tf->getLastChunk(),*tf,tor.getChunkSize());
fptr.seek(File::BEGIN,off);
fptr.read(tmp,tf->getLastChunkSize());
out.writeLastChunk(tmp,tf->getLastChunkSize());
}
delete [] tmp;
}
catch (...)
{
delete [] tmp;
throw;
}
}
void MultiFileCache::recreateFile(TorrentFile* tf,const TQString & dnd_file,const TQString & output_file)
{
DNDFile dnd(dnd_file);
// create the output file
bt::Touch(output_file);
// truncate it
try
{
bool res = false;
#ifdef HAVE_XFS_XFS_H
if( (! res) && (Settings::fullDiskPreallocMethod() == 1) )
{
res = XfsPreallocate(output_file, tf->getSize());
}
#endif
if(! res)
{
bt::TruncateFile(output_file,tf->getSize());
}
}
catch (bt::Error & e)
{
// first attempt failed, must be fat so try that
if (!FatPreallocate(output_file,tf->getSize()))
{
throw Error(i18n("Cannot preallocate diskspace : %1").arg(strerror(errno)));
}
}
Uint32 cs = 0;
if (tf->getFirstChunk() == tor.getNumChunks() - 1)
{
cs = tor.getFileLength() % tor.getChunkSize();
if (cs == 0)
cs = tor.getChunkSize();
}
else
cs = tor.getChunkSize();
File fptr;
if (!fptr.open(output_file,"r+b"))
throw Error(i18n("Cannot open file %1 : %2").arg(output_file).arg(fptr.errorString()));
Uint32 ts = cs - tf->getFirstChunkOffset() > tf->getLastChunkSize() ?
cs - tf->getFirstChunkOffset() : tf->getLastChunkSize();
Uint8* tmp = new Uint8[ts];
try
{
dnd.readFirstChunk(tmp,0,cs - tf->getFirstChunkOffset());
fptr.write(tmp,cs - tf->getFirstChunkOffset());
if (tf->getFirstChunk() != tf->getLastChunk())
{
Uint64 off = FileOffset(tf->getLastChunk(),*tf,tor.getChunkSize());
fptr.seek(File::BEGIN,off);
dnd.readLastChunk(tmp,0,tf->getLastChunkSize());
fptr.write(tmp,tf->getLastChunkSize());
}
delete [] tmp;
}
catch (...)
{
delete [] tmp;
throw;
}
}
void MultiFileCache::preallocateDiskSpace(PreallocationThread* prealloc)
{
Out() << "MultiFileCache::preallocateDiskSpace" << endl;
PtrMap<Uint32,CacheFile>::iterator i = files.begin();
while (i != files.end())
{
CacheFile* cf = i->second;
if (!prealloc->isStopped())
{
cf->preallocate(prealloc);
}
else
{
// we got interrupted tell the thread we are not finished and return
prealloc->setNotFinished();
return;
}
i++;
}
}
bool MultiFileCache::hasMissingFiles(TQStringList & sl)
{
bool ret = false;
for (Uint32 i = 0;i < tor.getNumFiles();i++)
{
TorrentFile & tf = tor.getFile(i);
if (tf.doNotDownload())
continue;
TQString p = cache_dir + tf.getPath();
TQFileInfo fi(p);
// always use symlink first, file might have been moved
if (!fi.exists())
{
ret = true;
p = fi.readLink();
if (p.isNull())
p = output_dir + tf.getPath();
sl.append(p);
tf.setMissing(true);
}
else
{
p = output_dir + tf.getPath();
// no symlink so try the actual file
if (!bt::Exists(p))
{
ret = true;
sl.append(p);
tf.setMissing(true);
}
}
}
return ret;
}
static void DeleteEmptyDirs(const TQString & output_dir,const TQString & fpath)
{
TQStringList sl = TQStringList::split(bt::DirSeparator(),fpath);
// remove the last, which is just the filename
sl.pop_back();
while (sl.count() > 0)
{
TQString path = output_dir;
// reassemble the full directory path
for (TQStringList::iterator itr = sl.begin(); itr != sl.end();itr++)
path += *itr + bt::DirSeparator();
TQDir dir(path);
TQStringList el = dir.entryList(TQDir::All|TQDir::System|TQDir::Hidden);
el.remove(".");
el.remove("..");
if (el.count() == 0)
{
// no childern so delete the directory
Out(SYS_GEN|LOG_IMPORTANT) << "Deleting empty directory : " << path << endl;
bt::Delete(path,true);
sl.pop_back(); // remove the last so we can go one higher
}
else
{
// children, so we cannot delete any more directories higher up
return;
}
}
// now the output_dir itself
TQDir dir(output_dir);
TQStringList el = dir.entryList(TQDir::All|TQDir::System|TQDir::Hidden);
el.remove(".");
el.remove("..");
if (el.count() == 0)
{
Out(SYS_GEN|LOG_IMPORTANT) << "Deleting empty directory : " << output_dir << endl;
bt::Delete(output_dir,true);
}
}
void MultiFileCache::deleteDataFiles()
{
for (Uint32 i = 0;i < tor.getNumFiles();i++)
{
TorrentFile & tf = tor.getFile(i);
TQString fpath = tf.getPath();
if (!tf.doNotDownload())
{
// first delete the file
bt::Delete(output_dir + fpath);
}
// check for subdirectories
DeleteEmptyDirs(output_dir,fpath);
}
}
Uint64 MultiFileCache::diskUsage()
{
Uint64 sum = 0;
for (Uint32 i = 0;i < tor.getNumFiles();i++)
{
TorrentFile & tf = tor.getFile(i);
if (tf.doNotDownload())
continue;
try
{
CacheFile* cf = files.find(i);
if (cf)
{
sum += cf->diskUsage();
}
else
{
// doesn't exist yet, must be before open is called
// so create one and delete it right after
cf = new CacheFile();
cf->open(cache_dir + tf.getPath(),tf.getSize());
sum += cf->diskUsage();
delete cf;
}
}
catch (bt::Error & err) // make sure we catch any exceptions
{
Out(SYS_DIO|LOG_DEBUG) << "Error: " << err.toString() << endl;
}
}
return sum;
}
///////////////////////////////
Uint64 FileOffset(Chunk* c,const TorrentFile & f,Uint64 chunk_size)
{
return FileOffset(c->getIndex(),f,chunk_size);
}
Uint64 FileOffset(Uint32 cindex,const TorrentFile & f,Uint64 chunk_size)
{
return f.fileOffset(cindex,chunk_size);
}
}
View Git Blame Copy Permalink