dak/tools/dsync-0.0/cmdline/dsync-flist.cc

// -*- mode: cpp; mode: fold -*-
// Description								/*{{{*/
// $Id: dsync-flist.cc,v 1.27 1999/12/26 06:59:00 jgg Exp $
/* ######################################################################

   Dsync FileList is a tool to manipulate and generate the dsync file 
   listing
   
   Several usefull functions are provided, the most notable is to generate
   the file list and to dump it. There is also a function to compare the
   file list against a local directory tree.
   
   ##################################################################### */
									/*}}}*/
// Include files							/*{{{*/
#ifdef __GNUG__
#pragma implementation "dsync-flist.h"
#endif 

#include "dsync-flist.h"
#include <dsync/cmndline.h>
#include <dsync/error.h>
#include <dsync/md5.h>
#include <dsync/strutl.h>

#include <config.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <utime.h>
#include <unistd.h>
#include <termios.h>
#include <signal.h>

#include <iostream>
using namespace std;

									/*}}}*/

// Externs								/*{{{*/
ostream c0out(cout.rdbuf());
ostream c1out(cout.rdbuf());
ostream c2out(cout.rdbuf());
ofstream devnull("/dev/null");
unsigned int ScreenWidth = 80;
									/*}}}*/

// Progress::Progress - Constructor					/*{{{*/
// ---------------------------------------------------------------------
/* */
Progress::Progress()
{
   Quiet = false;
   if (_config->FindI("quiet",0) > 0)
      Quiet = true;
   DirCount = 0;
   FileCount = 0;
   LinkCount = 0;
   Bytes = 0;
   CkSumBytes = 0;
   gettimeofday(&StartTime,0);
}
									/*}}}*/
// Progress::Done - Clear the progress meter				/*{{{*/
// ---------------------------------------------------------------------
/* */
void Progress::Done()
{
   if (Quiet == false)
      c0out << '\r' << BlankLine << '\r' << flush;
   BlankLine[0] = 0;
}
									/*}}}*/
// Progress::ElaspedTime - Return the time that has elapsed		/*{{{*/
// ---------------------------------------------------------------------
/* Computes the time difference with maximum accuracy */
double Progress::ElapsedTime()
{
   // Compute the CPS and elapsed time
   struct timeval Now;
   gettimeofday(&Now,0);

   return Now.tv_sec - StartTime.tv_sec + (Now.tv_usec - 
					   StartTime.tv_usec)/1000000.0;
}
									/*}}}*/
// Progress::Update - Update the meter					/*{{{*/
// ---------------------------------------------------------------------
/* */
void Progress::Update(const char *Directory)
{
   LastCount = DirCount+LinkCount+FileCount;
   
   if (Quiet == true)
      return;

   // Put the number of files and bytes at the end of the meter
   char S[1024];
   if (ScreenWidth > sizeof(S)-1)
      ScreenWidth = sizeof(S)-1;
   
   unsigned int Len = snprintf(S,sizeof(S),"|%lu %sb",
			       DirCount+LinkCount+FileCount,
			       SizeToStr(Bytes).c_str());
   
   memmove(S + (ScreenWidth - Len),S,Len+1);
   memset(S,' ',ScreenWidth - Len);
   
   // Put the directory name at the front, possibly shortened
   if (Directory == 0 || Directory[0] == 0)
      S[snprintf(S,sizeof(S),"<root>")] = ' ';
   else
   {
      // If the path is too long fix it and prefix it with '...'
      if (strlen(Directory) >= ScreenWidth - Len - 1)
      {
	 S[snprintf(S,sizeof(S),"%s",Directory + 
		    strlen(Directory) - ScreenWidth + Len + 1)] = ' ';
	 S[0] = '.'; S[1] = '.'; S[2] = '.';
      }
      else
	 S[snprintf(S,sizeof(S),"%s",Directory)] = ' ';
   }
   
   strcpy(LastLine,S);
   c0out << S << '\r' << flush;
   memset(BlankLine,' ',strlen(S));
   BlankLine[strlen(S)] = 0;
}
									/*}}}*/
// Progress::Stats - Show a statistics report				/*{{{*/
// ---------------------------------------------------------------------
/* */
void Progress::Stats(bool CkSum)
{
   // Display some interesting statistics
   double Elapsed = ElapsedTime();
   c1out << DirCount << " directories, " << FileCount <<
      " files and " << LinkCount << " links (" << 
      (DirCount+FileCount+LinkCount) << "). ";
   if (CkSum == true)
   {
      if (CkSumBytes == Bytes)
	 c1out << "Total Size is " << SizeToStr(Bytes) << "b. ";
      else
	 c1out << SizeToStr(CkSumBytes) << '/' <<
           SizeToStr(Bytes) << "b hashed.";
   }   
   else
      c1out << "Total Size is " << SizeToStr(Bytes) << "b. ";
      
   c1out << endl;
   c1out << "Elapsed time " <<  TimeToStr((long)Elapsed) <<
      " (" << SizeToStr((DirCount+FileCount+LinkCount)/Elapsed) <<
      " files/sec) ";
   if (CkSumBytes != 0)
      c1out << " (" << SizeToStr(CkSumBytes/Elapsed) << "b/s hash)";
   c1out << endl;
}
									/*}}}*/

// ListGenerator::ListGenerator - Constructor				/*{{{*/
// ---------------------------------------------------------------------
/* */
ListGenerator::ListGenerator()
{
   Act = !_config->FindB("noact",false);
   StripDepth = _config->FindI("FileList::CkSum-PathStrip",0);
   Verbose = false;
   if (_config->FindI("verbose",0) > 0)
      Verbose = true;
   DB = 0;
   DBIO = 0;

   // Set RSync checksum limits
   MinRSyncSize = _config->FindI("FileList::MinRSyncSize",0);
   if (MinRSyncSize == 0)
      MinRSyncSize = 1;
   if (_config->FindB("FileList::RSync-Hashes",false) == false)
       MinRSyncSize = 0;
       
   // Load the rsync filter
   if (RSyncFilter.LoadFilter(_config->Tree("FList::RSync-Filter")) == false)
      return;
       
   // Load the clean filter
   if (RemoveFilter.LoadFilter(_config->Tree("FList::Clean-Filter")) == false)
      return;
}
									/*}}}*/
// ListGenerator::~ListGenerator - Destructor				/*{{{*/
// ---------------------------------------------------------------------
/* */
ListGenerator::~ListGenerator()
{
   delete DB;
   delete DBIO;
}
									/*}}}*/
// ListGenerator::Visit - Collect statistics about the tree		/*{{{*/
// ---------------------------------------------------------------------
/* */
int ListGenerator::Visit(const char *Directory,const char *File,
			 struct stat const &Stat)
{
   if (Prog.DirCount+Prog.LinkCount+Prog.FileCount - Prog.LastCount > 100 ||
       File == 0)
      Prog.Update(Directory);
   
   // Ignore directory enters
   if (File == 0)
      return 0;
   
   // Increment our counters
   if (S_ISDIR(Stat.st_mode) != 0)
      Prog.DirCount++;
   else
   {
      if (S_ISLNK(Stat.st_mode) != 0)
	 Prog.LinkCount++;
      else
	 Prog.FileCount++;
   }
   
   // Normal file
   if (S_ISREG(Stat.st_mode) != 0)
      Prog.Bytes += Stat.st_size;
   
   // Look for files to erase
   if (S_ISDIR(Stat.st_mode) == 0 &&
       RemoveFilter.Test(Directory,File) == false)
   {
      Prog.Hide();
      c1out << "Unlinking " << Directory << File << endl;
      Prog.Show();
      
      if (Act == true && unlink(File) != 0)
      {
	 _error->Errno("unlink","Failed to remove %s%s",Directory,File);
	 return -1;
      }
      
      return 1;
   }
   
   return 0;
}			 
									/*}}}*/
// ListGenerator::EmitMD5 - Perform md5 lookup caching			/*{{{*/
// ---------------------------------------------------------------------
/* This looks up the file in the cache to see if it is one we already 
   know the hash too */
bool ListGenerator::EmitMD5(const char *Dir,const char *File,
			    struct stat const &St,unsigned char MD5[16],
			    unsigned int Tag,unsigned int Flag)
{
   if ((IO->Header.Flags[Tag] & Flag) != Flag)
      return true;

   // Lookup the md5 in the old file list
   if (DB != 0 && (DBIO->Header.Flags[Tag] & Flag) == Flag)
   {
      // Do a lookup and make sure the timestamps match
      dsFList List;
      bool Hit = false;
      const char *iDir = Dir;
      unsigned int Strip = StripDepth;
      while (true)
      {	 
	 if (DB->Lookup(*DBIO,iDir,File,List) == true && List.Entity != 0)
	 {
	    if ((signed)(List.Entity->ModTime + List.Head.Epoch) == St.st_mtime)
	       Hit = true;	    
	    break;
	 }
	 
	 if (Strip == 0)
	    break;
	 
	 Strip--;
	 for (; *iDir != 0 && *iDir != '/'; iDir++);
	 if (*iDir == 0 || iDir[1] == 0)
	    break;
	 iDir++;
      }
	 
      if (Hit == true)
      {
	 /* Both hardlinks and normal files have md5s, also check that the
	    sizes match */
	 if (List.File != 0 && List.File->Size == (unsigned)St.st_size)
	 {
	    memcpy(MD5,List.File->MD5,sizeof(List.File->MD5));
	    return true;
	 }
      }      
   }
   
   Prog.CkSumBytes += St.st_size;
   
   if (Verbose == true)
   {
      Prog.Hide();
      c1out << "MD5 " << Dir << File << endl;
      Prog.Show();
   }
   
   return dsGenFileList::EmitMD5(Dir,File,St,MD5,Tag,Flag);
}
									/*}}}*/
// ListGenerator::NeedsRSync - Check if a file is rsyncable		/*{{{*/
// ---------------------------------------------------------------------
/* This checks the rsync filter list and the rsync size limit*/
bool ListGenerator::NeedsRSync(const char *Dir,const char *File,
			       dsFList::NormalFile &F)
{
   if (MinRSyncSize == 0)
      return false;
   
   if (F.Size <= MinRSyncSize)
      return false;
   
   if (RSyncFilter.Test(Dir,File) == false)
      return false;
   
   /* Add it to the counters, EmitMD5 will not be called if rsync checksums
      are being built. */
   Prog.CkSumBytes += F.Size;  
   if (Verbose == true)
   {
      Prog.Hide();
      c1out << "RSYNC " << Dir << File << endl;
      Prog.Show();
   }
   
   return true;
}
									/*}}}*/

// Compare::Compare - Constructor					/*{{{*/
// ---------------------------------------------------------------------
/* */
Compare::Compare()
{
   Verbose = false;
   if (_config->FindI("verbose",0) > 0)
      Verbose = true;
   Act = !_config->FindB("noact",false);
   DoDelete = _config->FindB("delete",false);
}
									/*}}}*/
// Compare::Visit - Collect statistics about the tree			/*{{{*/
// ---------------------------------------------------------------------
/* */
bool Compare::Visit(dsFList &List,string Dir)
{
   if (Prog.DirCount+Prog.LinkCount+Prog.FileCount - Prog.LastCount > 100 ||
       List.Tag == dsFList::tDirStart)
      Prog.Update(Dir.c_str());
   
   // Increment our counters
   if (List.Tag == dsFList::tDirectory)
      Prog.DirCount++;
   else
   {
      if (List.Tag == dsFList::tSymlink)
	 Prog.LinkCount++;

      if (List.Tag == dsFList::tNormalFile || 
	  List.Tag == dsFList::tHardLink ||
	  List.Tag == dsFList::tDeviceSpecial)
	 Prog.FileCount++;
   }
   
   // Normal file
   if (List.File != 0)
      Prog.Bytes += List.File->Size;
   
   return true;
}
									/*}}}*/
// Compare::PrintPath - Print out a path string				/*{{{*/
// ---------------------------------------------------------------------
/* This handles the absolute paths that can occure while processing */
void Compare::PrintPath(ostream &out,string Dir,string Name)
{
   if (Name[0] != '/')
      out << Dir << Name << endl;
   else
      out << string(Name,Base.length()) << endl;
}
									/*}}}*/

// LookupPath - Find a full path within the database			/*{{{*/
// ---------------------------------------------------------------------
/* This does the necessary path simplification and symlink resolution
   to locate the path safely. The file must exist locally inorder to 
   resolve the local symlinks. */
bool LookupPath(const char *Path,dsFList &List,dsFileListDB &DB,
		dsFList::IO &IO)
{
   char Buffer[2024];
   strcpy(Buffer,Path);
      
   if (SimplifyPath(Buffer) == false || 
       ResolveLink(Buffer,sizeof(Buffer)) == false)
      return false;
   
   // Strip off the final component name
   char *I = Buffer + strlen(Buffer);
   for (; I != Buffer && (*I == '/' || *I == 0); I--);
   for (; I != Buffer && *I != '/'; I--);
   if (I != Buffer)
   {
      memmove(I+1,I,strlen(I) + 1);
      I++;
      *I = 0;
      I++;
      if (DB.Lookup(IO,Buffer,I,List) == false)
	 return false;
   }
   else
   {
      if (DB.Lookup(IO,"",I,List) == false)
	 return false;
   }
   
   return true;
}
									/*}}}*/
// PrintMD5 - Prints the MD5 of a file in the form similar to md5sum	/*{{{*/
// ---------------------------------------------------------------------
/* */
void PrintMD5(dsFList &List,const char *Dir,const char *File = 0)
{
   if (List.File == 0 || 
       List.Head.Flags[List.Tag] & dsFList::NormalFile::FlMD5 == 0)
      return;

   char S[16*2+1];
   for (unsigned int I = 0; I != 16; I++)
      sprintf(S+2*I,"%02x",List.File->MD5[I]);
   S[16*2] = 0;
   if (File == 0)
      cout << S << "  " << Dir << List.File->Name << endl;
   else
      cout << S << "  " << File << endl;
}
									/*}}}*/

// DoGenerate - The Generate Command					/*{{{*/
// ---------------------------------------------------------------------
/* */
bool DoGenerate(CommandLine &CmdL)
{
   ListGenerator Gen;
   if (_error->PendingError() == true)
      return false;
   
   // Load the filter list
   if (Gen.Filter.LoadFilter(_config->Tree("FileList::Filter")) == false)
      return false;

   // Load the delay filter list
   if (Gen.PreferFilter.LoadFilter(_config->Tree("FileList::Prefer-Filter")) == false)
      return false;
   
   // Determine the ordering to use
   string Ord = _config->Find("FileList::Order","tree");
   if (stringcasecmp(Ord,"tree") == 0)
      Gen.Type = dsGenFileList::Tree;
   else
   {
      if (stringcasecmp(Ord,"breadth") == 0)
	 Gen.Type = dsGenFileList::Breadth;
      else
      {
         if (stringcasecmp(Ord,"depth") == 0)
	    Gen.Type = dsGenFileList::Depth;
	 else
	    return _error->Error("Invalid ordering %s, must be tree, breadth or detph",Ord.c_str());
      }      
   }

   if (CmdL.FileList[1] == 0)
      return _error->Error("You must specify a file name");
   
   string List = CmdL.FileList[1];
   
   // Open the original file to pull cached Check Sums out of
   if (FileExists(List) == true && 
       _config->FindB("FileList::MD5-Hashes",false) == true)
   {
      Gen.DBIO = new dsMMapIO(List);
      if (_error->PendingError() == true)
	 return false;
      Gen.DB = new dsFileListDB;
      if (Gen.DB->Generate(*Gen.DBIO) == false)
	 return false;
   }   

   // Sub scope to close the file
   {      
      FdIO IO(List + ".new",FileFd::WriteEmpty);
      
      // Set the flags for the list
      if (_config->FindB("FileList::MD5-Hashes",false) == true)
      {
	 IO.Header.Flags[dsFList::tNormalFile] |= dsFList::NormalFile::FlMD5;
	 IO.Header.Flags[dsFList::tHardLink] |= dsFList::HardLink::FlMD5;
      }
      if (_config->FindB("FileList::Permissions",false) == true)
      {
	 IO.Header.Flags[dsFList::tDirectory] |= dsFList::Directory::FlPerm;
	 IO.Header.Flags[dsFList::tNormalFile] |= dsFList::NormalFile::FlPerm;
	 IO.Header.Flags[dsFList::tHardLink] |= dsFList::HardLink::FlPerm;
      }
      if (_config->FindB("FileList::Ownership",false) == true)
      {
	 IO.Header.Flags[dsFList::tDirectory] |= dsFList::Directory::FlOwner;
	 IO.Header.Flags[dsFList::tNormalFile] |= dsFList::NormalFile::FlOwner;
	 IO.Header.Flags[dsFList::tSymlink] |= dsFList::Symlink::FlOwner;
	 IO.Header.Flags[dsFList::tDeviceSpecial] |= dsFList::DeviceSpecial::FlOwner;
	 IO.Header.Flags[dsFList::tHardLink] |= dsFList::HardLink::FlOwner;
      }
      
      if (Gen.Go("./",IO) == false)
	 return false;
      Gen.Prog.Done();
      Gen.Prog.Stats(_config->FindB("FileList::MD5-Hashes",false));
      
      delete Gen.DB;
      Gen.DB = 0;
      delete Gen.DBIO;
      Gen.DBIO = 0;
   }
   
   // Just in case :>
   if (_error->PendingError() == true)
      return false;
   
   // Swap files
   bool OldExists = FileExists(List);
   if (OldExists == true && rename(List.c_str(),(List + "~").c_str()) != 0)
      return _error->Errno("rename","Unable to rename %s to %s~",List.c_str(),List.c_str());
   if (rename((List + ".new").c_str(),List.c_str()) != 0)
      return _error->Errno("rename","Unable to rename %s.new to %s",List.c_str(),List.c_str());
   if (OldExists == true && unlink((List + "~").c_str()) != 0)
      return _error->Errno("unlink","Unable to unlink %s~",List.c_str());
   
   return true;
}
									/*}}}*/
// DoDump - Dump the contents of a file list				/*{{{*/
// ---------------------------------------------------------------------
/* This displays a short one line dump of each record in the file */
bool DoDump(CommandLine &CmdL)
{
   if (CmdL.FileList[1] == 0)
      return _error->Error("You must specify a file name");
   
   // Open the file
   dsMMapIO IO(CmdL.FileList[1]);
   if (_error->PendingError() == true)
      return false;
   
   dsFList List;
   unsigned long CountDir = 0;
   unsigned long CountFile = 0;
   unsigned long CountLink = 0;
   unsigned long CountLinkReal = 0;
   unsigned long NumFiles = 0;
   unsigned long NumDirs = 0;
   unsigned long NumLinks = 0;
   double Bytes = 0;
   
   while (List.Step(IO) == true)
   {
      if (List.Print(cout) == false)
	 return false;

      switch (List.Tag)
      {
	 case dsFList::tDirMarker:
	 case dsFList::tDirStart:
	 case dsFList::tDirectory:
	 {
	    CountDir += List.Dir.Name.length();
	    if (List.Tag == dsFList::tDirectory)
	       NumDirs++;
	    break;
	 }

	 case dsFList::tHardLink:
	 case dsFList::tNormalFile:
	 {
	    CountFile += List.File->Name.length();
	    NumFiles++;
	    Bytes += List.File->Size;
	    break;
	 }
	 
	 case dsFList::tSymlink:
	 {
	    CountFile += List.SLink.Name.length();
	    CountLink += List.SLink.To.length();
	    
	    unsigned int Tmp = List.SLink.To.length();
	    if ((List.SLink.Compress & (1<<7)) == (1<<7))
	       Tmp -= List.SLink.Name.length();
	    Tmp -= List.SLink.Compress & 0x7F;
	    CountLinkReal += Tmp;
	    NumLinks++;
	    break;
	 }
      }
      if (List.Tag == dsFList::tTrailer)
	 break;
   }
   cout << "String Sizes: Dirs=" << CountDir << " Files=" << CountFile << 
      " Links=" << CountLink << " (" << CountLinkReal << ")";
   cout << " Total=" << CountDir+CountFile+CountLink << endl;
   cout << "Entries: Dirs=" << NumDirs << " Files=" << NumFiles << 
      " Links=" << NumLinks << " Total=" << NumDirs+NumFiles+NumLinks << endl;
   cout << "Totals " << SizeToStr(Bytes) << "b." << endl;
   
   return true;
}
									/*}}}*/
// DoMkHardLinks - Generate hardlinks for duplicated files		/*{{{*/
// ---------------------------------------------------------------------
/* This scans the archive for any duplicated files, it uses the MD5 of each
   file and searches a map for another match then links the two */
struct Md5Cmp
{
   unsigned char MD5[16];
   int operator <(const Md5Cmp &rhs) const {return memcmp(MD5,rhs.MD5,sizeof(MD5)) < 0;};
   int operator <=(const Md5Cmp &rhs) const {return memcmp(MD5,rhs.MD5,sizeof(MD5)) <= 0;};
   int operator >=(const Md5Cmp &rhs) const {return memcmp(MD5,rhs.MD5,sizeof(MD5)) >= 0;};
   int operator >(const Md5Cmp &rhs) const {return memcmp(MD5,rhs.MD5,sizeof(MD5)) > 0;};
   int operator ==(const Md5Cmp &rhs) const {return memcmp(MD5,rhs.MD5,sizeof(MD5)) == 0;};
   
   Md5Cmp(unsigned char Md[16]) {memcpy(MD5,Md,sizeof(MD5));};
};

struct Location
{
   string Dir;
   string File;
   
   Location() {};
   Location(string Dir,string File) : Dir(Dir), File(File) {};
};

bool DoMkHardLinks(CommandLine &CmdL)
{
   if (CmdL.FileList[1] == 0)
      return _error->Error("You must specify a file name");
   
   // Open the file
   dsMMapIO IO(CmdL.FileList[1]);
   if (_error->PendingError() == true)
      return false;

   dsFList List;
   if (List.Step(IO) == false || List.Tag != dsFList::tHeader)
      return _error->Error("Unable to read header");

   // Make sure we have hashes
   if ((IO.Header.Flags[dsFList::tNormalFile] & 
	dsFList::NormalFile::FlMD5) == 0 ||
       (IO.Header.Flags[dsFList::tHardLink] & 
	dsFList::HardLink::FlMD5) == 0)
      return _error->Error("The file list must contain MD5 hashes");
   
   string LastDir;
   double Savings = 0;
   unsigned long Hits = 0;
   bool Act = !_config->FindB("noact",false);   
   map<Md5Cmp,Location> Map;   
   while (List.Step(IO) == true)
   {
      // Entering a new directory, just store it..
      if (List.Tag == dsFList::tDirStart)
      {
	 LastDir = List.Dir.Name;
	 continue;
      }

      /* Handle normal file entities. Pre-existing hard links we treat
         exactly like a normal file, if two hard link chains are identical
	 one will be destroyed and its items placed on the other 
	 automatcially */
      if (List.File != 0)
      {
	 map<Md5Cmp,Location>::const_iterator I = Map.find(Md5Cmp(List.File->MD5));
	 if (I == Map.end())
	 {
	    Map[Md5Cmp(List.File->MD5)] = Location(LastDir,List.File->Name);
	    continue;
	 }

	 // Compute full file names for both
	 string FileA = (*I).second.Dir + (*I).second.File;
	 struct stat StA;
	 string FileB = LastDir + List.File->Name;
	 struct stat StB;
	 
	 // Stat them
	 if (lstat(FileA.c_str(),&StA) != 0)
	 {
	    _error->Warning("Unable to stat %s",FileA.c_str());
	    continue;
	 }	 
	 if (lstat(FileB.c_str(),&StB) != 0)
	 {
	    _error->Warning("Unable to stat %s",FileB.c_str());
	    continue;
	 }
	 
	 // Verify they are on the same filesystem
	 if (StA.st_dev != StB.st_dev || StA.st_size != StB.st_size)
	    continue;
	 
	 // And not merged..
	 if (StA.st_ino == StB.st_ino)
	    continue;
	 
	 c1out << "Dup " << FileA << endl;
         c1out << "    " << FileB << endl;
      
	 // Relink the file and copy the mod time from the oldest one.
	 if (Act == true)
	 {
	    if (unlink(FileB.c_str()) != 0)
	       return _error->Errno("unlink","Failed to unlink %s",FileB.c_str());
	    if (link(FileA.c_str(),FileB.c_str()) != 0)
	       return _error->Errno("link","Failed to link %s to %s",FileA.c_str(),FileB.c_str());
	    if (StB.st_mtime > StA.st_mtime)
	    {
	       struct utimbuf Time;
	       Time.actime = Time.modtime = StB.st_mtime;
	       if (utime(FileB.c_str(),&Time) != 0)
		  _error->Warning("Unable to set mod time for %s",FileB.c_str());	       
	    }
	 }
	 
	 // Counters
	 Savings += List.File->Size;
	 Hits++;
	 
	 continue;
      }
      
      if (List.Tag == dsFList::tTrailer)
	 break;
   }
   
   cout << "Total space saved by merging " << 
      SizeToStr(Savings) << "b. " << Hits << " files affected." << endl;
   return true;
}
									/*}}}*/
// DoLookup - Lookup a single file in the listing			/*{{{*/
// ---------------------------------------------------------------------
/* */
bool DoLookup(CommandLine &CmdL)
{
   if (CmdL.FileSize() < 4)
      return _error->Error("You must specify a file name, directory name and a entry");
   
   // Open the file
   dsMMapIO IO(CmdL.FileList[1]);
   if (_error->PendingError() == true)
      return false;

   // Index it
   dsFileListDB DB;
   if (DB.Generate(IO) == false)
      return false;

   dsFList List;
   if (DB.Lookup(IO,CmdL.FileList[2],CmdL.FileList[3],List) == false)
      return _error->Error("Unable to locate item");
   List.Print(cout);
   return true;
}
									/*}}}*/
// DoMD5Cache - Lookup a stream of files in the listing			/*{{{*/
// ---------------------------------------------------------------------
/* This takes a list of files names and prints out their MD5s, if possible
   data is used from the cache to save IO */
bool DoMD5Cache(CommandLine &CmdL)
{
   struct timeval Start;
   gettimeofday(&Start,0);
   
   if (CmdL.FileList[1] == 0)
      return _error->Error("You must specify a file name");
   
   // Open the file
   dsMMapIO IO(CmdL.FileList[1]);
   if (_error->PendingError() == true)
      return false;

   dsFList List;
   if (List.Step(IO) == false || List.Tag != dsFList::tHeader)
      return _error->Error("Unable to read header");
   
   // Make sure we have hashes
   if ((IO.Header.Flags[dsFList::tNormalFile] & 
	dsFList::NormalFile::FlMD5) == 0 ||
       (IO.Header.Flags[dsFList::tHardLink] & 
	dsFList::HardLink::FlMD5) == 0)
      return _error->Error("The file list must contain MD5 hashes");

   // Index it
   dsFileListDB DB;
   if (DB.Generate(IO) == false)
      return false;

   // Counters
   double Bytes = 0;
   double MD5Bytes = 0;
   unsigned long Files = 0;
   unsigned long Errors = 0;

   while (!cin == false)
   {
      char Buf2[200];
      cin.getline(Buf2,sizeof(Buf2));
      if (Buf2[0] == 0)
	 continue;
      Files++;
      
      // Stat the file
      struct stat St;
      if (stat(Buf2,&St) != 0)
      {
	 cout << "<ERROR> " << Buf2 << "(stat)" << endl;
	 Errors++;
	 continue;
      }
            
      // Lookup in the cache and make sure the file has not changed
      if (LookupPath(Buf2,List,DB,IO) == false ||
	  (signed)(List.Entity->ModTime + List.Head.Epoch) != St.st_mtime ||
	  (List.File != 0 && List.File->Size != (unsigned)St.st_size))
      {
	 _error->DumpErrors();
	 
	 // Open the file and hash it
	 MD5Summation Sum;
	 FileFd Fd(Buf2,FileFd::ReadOnly);
	 if (_error->PendingError() == true)
	 {
	    cout << "<ERROR> " << Buf2 << "(open)" << endl;
	    continue;
	 }
	 
	 if (Sum.AddFD(Fd.Fd(),Fd.Size()) == false)
	 {
	    cout << "<ERROR> " << Buf2 << "(md5)" << endl;
	    continue;
	 }
	 	 
	 // Store the new hash
	 List.Tag = dsFList::tNormalFile;
	 Sum.Result().Value(List.File->MD5);
	 List.File->Size = (unsigned)St.st_size;
	 
	 MD5Bytes += List.File->Size;
      }

      PrintMD5(List,0,Buf2);
      Bytes += List.File->Size;
   }

   // Print out a summary
   struct timeval Now;
   gettimeofday(&Now,0);
   double Delta = Now.tv_sec - Start.tv_sec + (Now.tv_usec - Start.tv_usec)/1000000.0;
   cerr << Files << " files, " << SizeToStr(MD5Bytes) << "/" << 
      SizeToStr(Bytes) << " MD5'd, " << TimeToStr((unsigned)Delta) << endl;;
      
   return true;
}
									/*}}}*/
// DoMD5Dump - Dump the md5 list					/*{{{*/
// ---------------------------------------------------------------------
/* This displays a short one line dump of each record in the file */
bool DoMD5Dump(CommandLine &CmdL)
{
   if (CmdL.FileList[1] == 0)
      return _error->Error("You must specify a file name");
   
   // Open the file
   dsMMapIO IO(CmdL.FileList[1]);
   if (_error->PendingError() == true)
      return false;
   
   dsFList List;
   if (List.Step(IO) == false || List.Tag != dsFList::tHeader)
      return _error->Error("Unable to read header");
   
   // Make sure we have hashes
   if ((IO.Header.Flags[dsFList::tNormalFile] & 
	dsFList::NormalFile::FlMD5) == 0 ||
       (IO.Header.Flags[dsFList::tHardLink] & 
	dsFList::HardLink::FlMD5) == 0)
      return _error->Error("The file list must contain MD5 hashes");
   
   string Dir;
   while (List.Step(IO) == true)
   {
      if (List.Tag == dsFList::tDirStart)
      {
	 Dir = List.Dir.Name;
	 continue;
      }
      
      PrintMD5(List,Dir.c_str());
      
      if (List.Tag == dsFList::tTrailer)
	 break;
   }   
   return true;
}
									/*}}}*/
// DoVerify - Verify the local tree against a file list			/*{{{*/
// ---------------------------------------------------------------------
/* */
bool DoVerify(CommandLine &CmdL)
{
   if (CmdL.FileList[1] == 0)
      return _error->Error("You must specify a file name");
   
   // Open the file
   dsMMapIO IO(CmdL.FileList[1]);
   if (_error->PendingError() == true)
      return false;
   
   /* Set the hashing type, we can either do a full verify or only a date
      check verify */
   Compare Comp;
   if (_config->FindB("FileList::MD5-Hashes",false) == true)
      Comp.HashLevel = dsDirCompare::Md5Always;
   else
      Comp.HashLevel = dsDirCompare::Md5Date;
   
   // Scan the file list
   if (Comp.Process(".",IO) == false)
      return false;
   Comp.Prog.Done();
   
   // Report stats
   Comp.Prog.Stats((IO.Header.Flags[dsFList::tNormalFile] & dsFList::NormalFile::FlMD5) != 0 ||
		   (IO.Header.Flags[dsFList::tHardLink] & dsFList::HardLink::FlMD5) != 0);
   
   return true;
}
									/*}}}*/
// SigWinch - Window size change signal handler				/*{{{*/
// ---------------------------------------------------------------------
/* */
void SigWinch(int)
{
   // Riped from GNU ls
#ifdef TIOCGWINSZ
   struct winsize ws;
  
   if (ioctl(1, TIOCGWINSZ, &ws) != -1 && ws.ws_col >= 5)
      ScreenWidth = ws.ws_col - 1;
   if (ScreenWidth > 250)
      ScreenWidth = 250;
#endif
}
									/*}}}*/
// ShowHelp - Show the help screen					/*{{{*/
// ---------------------------------------------------------------------
/* */
bool ShowHelp(CommandLine &CmdL)
{
   cout << PACKAGE << ' ' << VERSION << " for " << ARCHITECTURE <<
      " compiled on " << __DATE__ << "  " << __TIME__ << endl;
   
   cout << 
      "Usage: dsync-flist [options] command [file]\n"
      "\n"
      "dsync-flist is a tool for manipulating dsync binary file lists.\n"
      "It can generate the lists and check them against a tree.\n"
      "\n"
      "Commands:\n"
      "   generate - Build a file list\n"
      "   help - This help text\n"
      "   dump - Display the contents of the list\n"
      "   md5sums - Print out 'indices' file, suitable for use with md5sum\n"
      "   md5cache - Print out md5sums of the files given on stdin\n"
      "   link-dups - Look for duplicate files\n"
      "   lookup - Display a single file record\n"
      "   verify - Compare the file list against the local directory\n"
      "\n"   
      "Options:\n"
      "  -h  This help text.\n"
      "  -q  Loggable output - no progress indicator\n"
      "  -qq No output except for errors\n"
      "  -i=? Include pattern\n"
      "  -e=? Exclude pattern\n"
      "  -c=? Read this configuration file\n"
      "  -o=? Set an arbitary configuration option, ie -o dir::cache=/tmp\n"
      "See the dsync-flist(1) and dsync.conf(5) manual\n"
      "pages for more information." << endl;
   return 100;
}
									/*}}}*/

int main(int argc, const char *argv[])
{
   CommandLine::Args Args[] = {
      {'h',"help","help",0},
      {'q',"quiet","quiet",CommandLine::IntLevel},
      {'q',"silent","quiet",CommandLine::IntLevel},
      {'i',"include","FileList::Filter:: + ",CommandLine::HasArg},
      {'e',"exclude","FileList::Filter:: - ",CommandLine::HasArg},
      {'n',"no-act","noact",0},
      {'v',"verbose","verbose",CommandLine::IntLevel},
      {0,"delete","delete",0},
      {0,"prefer-include","FileList::Prefer-Filter:: + ",CommandLine::HasArg},
      {0,"prefer-exclude","FileList::Prefer-Filter:: - ",CommandLine::HasArg},
      {0,"pi","FileList::Prefer-Filter:: + ",CommandLine::HasArg},
      {0,"pe","FileList::Prefer-Filter:: - ",CommandLine::HasArg},
      {0,"clean-include","FList::Clean-Filter:: + ",CommandLine::HasArg},
      {0,"clean-exclude","FList::Clean-Filter:: - ",CommandLine::HasArg},
      {0,"ci","FList::Clean-Filter:: + ",CommandLine::HasArg},
      {0,"ce","FList::Clean-Filter:: - ",CommandLine::HasArg},
      {0,"rsync-include","FList::RSync-Filter:: + ",CommandLine::HasArg},
      {0,"rsync-exclude","FList::RSync-Filter:: - ",CommandLine::HasArg},
      {0,"ri","FList::RSync-Filter:: + ",CommandLine::HasArg},
      {0,"re","FList::RSync-Filter:: - ",CommandLine::HasArg},
      {0,"md5","FileList::MD5-Hashes",0},
      {0,"rsync","FileList::RSync-Hashes",0},
      {0,"rsync-min","FileList::MinRSyncSize",CommandLine::HasArg},
      {0,"perm","FileList::Permissions",0},
      {0,"owner","FileList::Ownership",0},
      {0,"order","FileList::Order",CommandLine::HasArg},
      {'c',"config-file",0,CommandLine::ConfigFile},
      {'o',"option",0,CommandLine::ArbItem},
      {0,0,0,0}};
   CommandLine::Dispatch Cmds[] = {{"generate",&DoGenerate},
                                   {"help",&ShowHelp},
                                   {"dump",&DoDump},
                                   {"link-dups",&DoMkHardLinks},
                                   {"md5sums",&DoMD5Dump},
                                   {"md5cache",&DoMD5Cache},
                                   {"lookup",&DoLookup},
                                   {"verify",&DoVerify},
                                   {0,0}};
   CommandLine CmdL(Args,_config);
   if (CmdL.Parse(argc,argv) == false)
   {
      _error->DumpErrors();
      return 100;
   }
   
   // See if the help should be shown
   if (_config->FindB("help") == true ||
       CmdL.FileSize() == 0)
      return ShowHelp(CmdL);   

   // Setup the output streams
/*   c0out.rdbuf(cout.rdbuf());
   c1out.rdbuf(cout.rdbuf());
   c2out.rdbuf(cout.rdbuf()); */
   if (_config->FindI("quiet",0) > 0)
      c0out.rdbuf(devnull.rdbuf());
   if (_config->FindI("quiet",0) > 1)
      c1out.rdbuf(devnull.rdbuf());

   // Setup the signals
   signal(SIGWINCH,SigWinch);
   SigWinch(0);
   
   // Match the operation
   CmdL.DispatchArg(Cmds);
   
   // Print any errors or warnings found during parsing
   if (_error->empty() == false)
   {
      
      bool Errors = _error->PendingError();
      _error->DumpErrors();
      return Errors == true?100:0;
   }
         
   return 0; 
}