#! /usr/bin/env python

#    cfv - Command-line File Verify
#    Copyright (C) 2000-2002  Matthew Mueller <donut@azstarnet.com>
#
#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

import getopt, re, os, string, sys, errno, time, copy

cftypes={}

LISTOK=512
LISTBAD=1024
LISTNOTFOUND=2048
LISTUNVERIFIED=4096
LISTARGS={'ok':LISTOK, 'bad':LISTBAD, 'notfound':LISTNOTFOUND, 'unverified':LISTUNVERIFIED}

def chomp(line):
	if line[-2:] == '\r\n': return line[:-2]
	elif line[-1:] in '\r\n': return line[:-1]
	return line


curdir=os.getcwd()
reldir=['']
prevdir=[]
def chdir(d):
	global curdir
	prevdir.append(curdir)
	reldir.append(os.path.join(reldir[-1], d))
	os.chdir(d)
	curdir=os.getcwd()
def cdup():
	global curdir
	reldir.pop()
	curdir=prevdir.pop()
	os.chdir(curdir)


def _pstd(s,nl='\n',write=sys.stdout.write):
	write(s + nl)
def _perr(s,nl='\n',write=sys.stderr.write):
	write(s + nl)

p=_pstd
def pverbose(s,nl='\n'):
	if config.verbose>0:
		p(s,nl)
def pinfo(s,nl='\n'):
	if config.verbose>=0 or config.verbose==-3:
		p(s,nl)
def perror(s,nl='\n'):
	if config.verbose>=-1:
		_perr(s,nl)

def plistf(filename):
	sys.stdout.write(perhaps_showpath(filename)+config.listsep)


class CFVException(Exception):
	pass

class CFVValueError(CFVException):#invalid argument in user input
	pass

class CFVNameError(CFVException):#invalid command in user input
	pass

class CFVSyntaxError(CFVException):#error in user input
	pass

class CFVVerifyError(CFVException):#invalid crc/filesize/etc
	pass

class Stats:
	def __init__(self):
		self.num=0
		self.ok=0
		self.badsize=0
		self.badcrc=0
		self.notfound=0
		self.ferror=0
		self.cferror=0
		self.bytesread=0L #use long int for size, to avoid possible overflow.
		self.unverifiedfiles=[]
		self.unverified=0
		self.diffcase=0
		self.quoted=0
		self.starttime=time.time()
		self.subcount=0

	def make_sub_stats(self):
		b = copy.copy(self)
		b.starttime = time.time()
		self.subcount = self.subcount + 1
		return b
	
	def sub_stats_end(self, end):
		for v in 'badcrc', 'badsize', 'bytesread', 'cferror', 'diffcase', 'ferror', 'notfound', 'num', 'ok', 'quoted', 'unverified':
			setattr(self, v, getattr(end, v) - getattr(self, v))

	def print_stats(self):
		pinfo('%i files'%self.num,'')
		pinfo(', %i OK' %self.ok,'')
		if self.badcrc:
			pinfo(', %i badcrc' %self.badcrc,'')
		if self.badsize:
			pinfo(', %i badsize' %self.badsize,'')
		if self.notfound:
			pinfo(', %i not found' %self.notfound,'')
		if self.ferror:
			pinfo(', %i file errors' %self.ferror,'')
		if self.unverified:
			pinfo(', %i unverified' %self.unverified,'')
		if self.cferror:
			pinfo(', %i chksum file errors' %self.cferror,'')
		if self.diffcase:
			pinfo(', %i differing cases' %self.diffcase,'')
		if self.quoted:
			pinfo(', %i quoted filenames' %self.quoted,'')

		elapsed=time.time()-self.starttime
		pinfo('.  %.3f seconds, '%(elapsed),'')
		if elapsed==0.0:
			pinfo('%.1fK'%(self.bytesread/1024.0),'')
		else:
			pinfo('%.1fK/s'%(self.bytesread/elapsed/1024.0),'')

		pinfo('\n','')

class Config:
	verbose=0 # -1=quiet  0=norm  1=noisy
	docrcchecks=1
	dirsort=1
	cmdlinesort=1
	cmdlineglob='a'
	recursive=0
	showunverified=0
	defaulttype='sfv'
	ignorecase=0
	fixpaths=''
	strippaths=0
	showpaths=2
	showpathsabsolute=0
	gzip=0
	rename=0
	renameformat='%(name)s.bad-%(count)i%(ext)s'
	renameformatnocount=0
	list=0
	listsep='\n'
	def setdefault(self,cftype):
		if cftype in cftypes.keys():
			self.defaulttype=cftype
		else:
			raise CFVValueError, "invalid default type '%s'"%cftype
	def setintr(self,o,v,min,max):
		try:
			x=int(v)
			if x>max or x<min:
				raise CFVValueError, "out of range int '%s' for %s"%(v,o)
			self.__dict__[o]=x
		except ValueError:
			raise CFVValueError, "invalid int type '%s' for %s"%(v,o)
	def setbool(self,o,v):
		v=string.lower(v)
		if v in ('yes','on','true','1'):
			x=1
		elif v in ('no','off','false','0'):
			x=0
		else:
			raise CFVValueError, "invalid bool type '%s' for %s"%(v,o)
		self.__dict__[o]=x
	def setstr(self,o,v):
		self.__dict__[o]=v
	def setx(self,o,v):
		if o=="default":
			self.setdefault(v)
		elif o in ("dirsort","cmdlinesort","showunverified","ignorecase","rename"):
			self.setbool(o,v)
		elif o=="cmdlineglob":
			if 'yes'.startswith(v.lower()):
				self.cmdlineglob='y'
			elif 'auto'.startswith(v.lower()):
				self.cmdlineglob='a'
			elif 'no'.startswith(v.lower()):
				self.cmdlineglob='n'
			else:
				raise CFVValueError, "invalid cmdlineglob option '%s', must be 'no', 'auto', or 'yes'"%v
		elif o=="verbose":
			try:
				self.setintr(o,v,-3,1)
			except CFVValueError:
				if   v=='v':  self.verbose=1
				elif v=='V':  self.verbose=0
				elif v=='VV': self.verbose=-1
				elif v=='q':  self.verbose=-2
				elif v=='Q':  self.verbose=-3
				else:
					raise CFVValueError, "invalid verbose option '%s', must be 'v', 'V', 'VV', 'q', 'Q' or -3 - 1"%v
		elif o in ("gzip",):
			self.setintr(o,v,-1,1)
		elif o in ("recursive",):
			self.setintr(o,v,0,2)
		elif o=="showpaths":
			p=0
			a=0
			for v in v.split('-'):
				if not p:
					if 'none'.startswith(v.lower()) or v=='0':
						self.showpaths=0
						p=1; continue
					elif 'auto'.startswith(v.lower()) or v=='2':
						self.showpaths=2
						p=1; continue
					elif 'yes'.startswith(v.lower()) or v=='1':
						self.showpaths=1
						p=1; continue
				if not a:
					if 'absolute'.startswith(v.lower()):
						self.showpathsabsolute=1
						a=1; continue
					elif 'relative'.startswith(v.lower()):
						self.showpathsabsolute=0
						a=1; continue
				raise CFVValueError, "invalid showpaths option '%s', must be 'none', 'auto', 'yes', 'absolute', or 'relative'"%v
		elif o=="strippaths":
			if 'none'.startswith(v.lower()):
				self.strippaths='n'
			elif 'all'.startswith(v.lower()):
				self.strippaths='a'
			else:
				try:
					x=int(v)
					if x<0:
						raise ValueError
					self.strippaths=x
				except ValueError:
					raise CFVValueError, "invalid strippaths option '%s', must be 'none', 'all', or int >=0"%v
		elif o=="fixpaths":
			self.setstr(o,v)
		elif o=="renameformat":
			testmap=make_rename_formatmap('1.2')
			testmapwc=make_rename_formatmap('1.2')
			testmapwc['count']=1
			format_test=v%testmapwc
			try:
				format_test=v%testmap
				self.renameformatnocount=1 #if we can get here, it doesn't use the count param
			except KeyError:
				self.renameformatnocount=0
			self.renameformat=v
		else:
			raise CFVNameError, "invalid option '%s'"%o
	def readconfig(self):
		filename=os.path.expanduser("~/.cfvrc")
		if os.path.isfile(filename):
			file=open(filename,"r")
			l=0
			while 1:
				l=l+1
				s=file.readline()
				if not len(s):
					break #end of file
				if s[0]=="#":
					continue #ignore lines starting with #
				#s=re.sub('[\n\r]','',s)
				s=chomp(s)
				if not len(s):
					continue #ignore blank lines
				x=re.search("^(.*) (.*)$",s)
				if not x:
					raise CFVSyntaxError, "%s:%i: invalid line '%s'"%(filename,l,s)
				else:
					o,v=x.groups()
					try:
						self.setx(o,v)
					except CFVException, err:
						raise sys.exc_info()[0], "%s:%i: %s"%(filename,l,err), sys.exc_info()[2] #reuse the traceback of the original exception, but add file and line numbers to the error
	def __init__(self):
		self.readconfig()
			
			
def make_rename_formatmap(l_filename):
	sp=os.path.splitext(l_filename)
	return {'name':sp[0], 'ext':sp[1], 'fullname':l_filename}

version='1.11'#-pre'+'$Revision: 1.65 $'[11:-2]

try:
	if os.environ.get('CFV_NOFCHKSUM'): raise ImportError
	import fchksum
	try:
		if fchksum.version()<2:raise ImportError
	except:
		_perr("old fchksum version installed, using std python modules. please update.") #can't use perror yet since config hasn't been done..
		raise ImportError
	def getfilemd5(file):
		c,s=fchksum.fmd5t(file)
		stats.bytesread=stats.bytesread+s
		return c,s
	def getfilecrc(file):
		c,s=fchksum.fcrc32t(file)
		stats.bytesread=stats.bytesread+s
		return c,s
except ImportError:
	import md5,zlib
	try:
		if os.environ.get('CFV_NOMMAP'): raise ImportError
		import mmap
		if hasattr(mmap, 'ACCESS_READ'):
			def dommap(fileno, len):#generic mmap.  python2.2 adds ACCESS_* args that work on both nix and win.
				if len==0: return '' #mmap doesn't like length=0
				return mmap.mmap(fileno, len, access=mmap.ACCESS_READ)
		elif hasattr(mmap, 'PROT_READ'):
			def dommap(fileno, len):#unix mmap.  python default is PROT_READ|PROT_WRITE, but we open readonly.
				if len==0: return '' #mmap doesn't like length=0
				return mmap.mmap(fileno, len, mmap.MAP_SHARED, mmap.PROT_READ)
		else:
			def dommap(fileno, len):#windows mmap.
				if len==0: return ''
				return mmap.mmap(fileno, len)
		nommap=0
	except ImportError:
		nommap=1

	def getfilemd5(file):
		if file=='':
			f=sys.stdin
		else:
			f=open(file,'rb')
		if f==sys.stdin or nommap:
			m=md5.new()
			s=0L
			while 1:
				x=f.read(65536)
				if not len(x):
					c = "%02x"*16 % tuple(map(ord, m.digest())) #m.hexdigest() isn't available till python2.0, but this isn't too bad. (seen on c.l.p)
					stats.bytesread=stats.bytesread+s
					return c,s
				s=s+len(x)
				m.update(x)
		else:
			s = os.path.getsize(file)
			m = md5.new(dommap(f.fileno(), s))
			c = "%02x"*16 % tuple(map(ord, m.digest()))
			stats.bytesread = stats.bytesread+s
			return c,s
			
	def getfilecrc(file):
		if file=='':
			f=sys.stdin
		else:
			f=open(file,'rb')
		if f==sys.stdin or nommap:
			c=zlib.crc32('')
			s=0L
			while 1:
				x=f.read(65536)
				if not len(x):
					stats.bytesread=stats.bytesread+s
					return "%08X"%c,s
				s=s+len(x)
				c=zlib.crc32(x,c)
		else:
			s = os.path.getsize(file)
			c = zlib.crc32(dommap(f.fileno(), s))
			stats.bytesread = stats.bytesread+s
			return "%08X"%c,s

try:
	import filecmp
	def fcmp(f1, f2):
		return filecmp.cmp(f1, f2, shallow=0)
except ImportError:
	import cmp #obsolete since python 1.6
	def fcmp(f1, f2):
		#old cmp module is always shallow, so call its real do_cmp function if the files signatures match.  do_cmp is undocumented but in all the versions of the cmp.py on sourceforge python cvs so I think its safe :)
		return cmp.cmp(f1, f2) and cmp.do_cmp(f1, f2)

try:
	staticmethod #new in python 2.2
except NameError:
	class staticmethod:
		def __init__(self, anycallable):
			self.__call__ = anycallable
					

class PeekFile:
	def __init__(self, fileobj, filename=None, doclose=1):
		self.fobj = fileobj
		self.peekbuf = ''
		self.doclose = doclose

		self.name=filename or fileobj.name
		self.setpoked()
		self.write=self.fobj.write
		self.flush=self.fobj.flush
	def close(self):
		if self.doclose:
			self.fobj.close()
		else:
			self.flush()
	def setpoked(self):
		if self.peekbuf:
			self.read = self.p_read
			self.readline = self.p_readline
		else:
			self.read = self.fobj.read
			self.readline = self.fobj.readline
	def peek(self, size):
		if len(self.peekbuf) < size:
			self.peekbuf = self.peekbuf + self.fobj.read(size - len(self.peekbuf))
		self.setpoked()
		return self.peekbuf[:size]
	def hackedreadline(self, size=-1):
		try:
			return self.fobj.readline(size)
		except TypeError:
			return self.fobj.readline() #hack for broken readline() in older gzip.py versions
	def peekline(self, size=-1):
		lines = self.peekbuf.splitlines(1)
		if size >= 0 and len(self.peekbuf) >= size:
			return lines[0][:size]
		if len(lines) > 1 or '\n' in self.peekbuf or '\r' in self.peekbuf:
			return lines[0]
		if size >= 0:
			self.peekbuf = self.peekbuf + self.hackedreadline(size - len(self.peekbuf))
		else:
			self.peekbuf = self.peekbuf + self.fobj.readline()
		self.setpoked()
		return self.peekbuf
	def p_read(self, size=-1):
		if size == -1:
			ret = self.peekbuf
			self.peekbuf = ''
			self.setpoked()
			return ret + self.read()
		if size < len(self.peekbuf):
			ret = self.peekbuf[:size]
			self.peekbuf = self.peekbuf[size:]
			self.setpoked()
			return ret
		ret = self.peekbuf
		self.peekbuf = ''
		self.setpoked()
		return ret + self.read(size - len(ret))
	def p_readline(self, size=-1): #note that on dos formatted files, this function is not 100% safe, if you peek the exact number of chars to cut the CRLF in half, then the LF will appear to be a empty line after this one... (sigh)
		lines = self.peekbuf.splitlines(1)
		if size >= 0 and len(self.peekbuf) >= size:
			self.peekbuf = lines[0][size:] + ''.join(lines[1:])
			self.setpoked()
			return lines[0][:size]
		if len(lines) > 1 or '\n' in self.peekbuf or '\r' in self.peekbuf:
			self.peekbuf = ''.join(lines[1:])
			self.setpoked()
			return lines[0]
		self.peekbuf = ''
		self.setpoked()
		if size >=0 :
			return lines[0] + self.hackedreadline(size - len(self.peekbuf))
		else:
			return lines[0] + self.fobj.readline()


def doopen(filename,mode='r'):
	if mode[0]=='r' and string.find(mode,'b')<0:
		mode=mode+'b' #read all files in binary mode (since .pars are binary, and we don't always know the filetype when opening, just open everything binary.  The text routines should cope with all types of line endings anyway, so this doesn't hurt us.)
	if filename=='-':
		fileobj=(mode[0]=='r' and sys.stdin or sys.stdout)
	if config.gzip>=2 or (config.gzip>=0 and string.lower(filename[-3:])=='.gz'):
		import gzip
		if string.find(mode,'b')<0:
			mode=mode+'b' #open gzip files in binary mode
		if filename=='-':
			if mode[0]=='r':
				import StringIO
				return PeekFile(gzip.GzipFile(mode=mode,fileobj=StringIO.StringIO(fileobj.read())), filename) #lovely hack since gzip.py requires a bunch of seeking.. bleh.
			return PeekFile(gzip.GzipFile(mode=mode,fileobj=fileobj), filename)
		return PeekFile(gzip.open(filename,mode), filename)
	else:
		if filename=='-':
			#return fileobj
			#return os.fdopen(fileobj.fileno(),mode) #eek, even worse.
			return PeekFile(fileobj,doclose=0)
		return PeekFile(open(filename,mode))


class ChksumType:
	def __init__(self, testfiles):
		self.testfiles = testfiles
	
	def auto_filename_match(filename):
		return 0
	auto_filename_match = staticmethod(auto_filename_match)

	def test_chksumfile(self,file,filename):
		try:
			if config.verbose>=0 or config.verbose==-3:
				cf_stats = stats.make_sub_stats()
			if not file:
				file=doopen(filename,'r')
			self.do_test_chksumfile(file)
			if config.verbose>=0 or config.verbose==-3:
				cf_stats.sub_stats_end(stats)
				pinfo(perhaps_showpath(file.name)+': ','')
				cf_stats.print_stats()
		except EnvironmentError, a:
			stats.cferror=stats.cferror+1
			perror('%s : %s (CF)'%(perhaps_showpath(file.name),a[1]))

	def do_test_chksumfile_print_testingline(self, file):
		pverbose('testing from %s (%s)'%(file.name, string.lower(self.__class__.__name__)))

	def do_test_chksumfile(self, file):
		self.do_test_chksumfile_print_testingline(file)
		line=1
		while 1:
			l=file.readline()
			if not len(l):
				break
			if self.do_test_chksumline(l):
				stats.cferror=stats.cferror+1
				perror('%s : unrecognized line %i (CF)'%(perhaps_showpath(file.name),line))
			line=line+1

	def test_file(self,filename,filecrc,filesize=-1):
		if config.fixpaths:
			filename=fixpath(filename)
		filename=os.path.normpath(filename)
		if config.strippaths!='n':
			filename=strippath(filename, config.strippaths)
		if self.testfiles:
			if config.ignorecase:
				if string.lower(filename) not in self.testfiles:
					return
			elif filename not in self.testfiles:
				return
		stats.num=stats.num+1
		l_filename=filename
		try:
			if config.ignorecase:
				#we need to find the correct filename if using showunverified, even if the filename we are given works, since on FAT/etc filesystems the incorrect case will still return true from os.path.exists, but it could be the incorrect case to remove from the unverified files list.
				if config.showunverified or not os.path.exists(l_filename):
					try:
						l_filename=nocase_findfile(l_filename)
					except IOError, e:
						if e[0]==errno.ENOENT and l_filename[0]=='"' and l_filename[-1]=='"':
							stats.quoted = stats.quoted + 1
							l_filename=nocase_findfile(l_filename[1:-1])
						else:
							raise
			elif l_filename[0]=='"' and l_filename[-1]=='"' and not os.path.exists(l_filename):
				l_filename=l_filename[1:-1] #work around buggy sfv encoders that quote filenames
				stats.quoted = stats.quoted + 1
			if config.showunverified:
				try:
					stats.unverifiedfiles[-1].remove(l_filename)
				except ValueError:
					pass #don't error if file has already been removed.
			if filesize>=0:
				fs=os.path.getsize(l_filename)
				if fs!=filesize:
					stats.badsize=stats.badsize+1
					raise CFVVerifyError, 'file size does not match (%s!=%i)'%(filesize,fs)
			if config.docrcchecks and filecrc!=None:
				c=self.do_test_file(l_filename,filecrc)
				if c:
					stats.badcrc=stats.badcrc+1
					raise CFVVerifyError, 'crc does not match (%s!=%s)'%(filecrc,c)
			else:
				if not os.path.exists(l_filename):
					raise EnvironmentError, (errno.ENOENT,"missing")
				if not os.path.isfile(l_filename):
					raise EnvironmentError, (errno.ENOENT,"not a file")
				filecrc="exists" #since we didn't actually test the crc, make verbose mode merely say it exists
		except EnvironmentError, a:
			if a[0]==errno.ENOENT:
				stats.notfound=stats.notfound+1
				if config.list&LISTNOTFOUND:
					plistf(l_filename)
			else:
				stats.ferror=stats.ferror+1
			perror('%s : %s'%(perhaps_showpath(l_filename),a[1]))
			return -1
		except CFVVerifyError, a:
			reninfo=''
			if config.list&LISTBAD:
				plistf(l_filename)
			if config.rename:
				formatmap=make_rename_formatmap(l_filename) 
				for count in xrange(0,sys.maxint):
					formatmap['count']=count
					newfilename=config.renameformat%formatmap
					if config.renameformatnocount and count>0:
						newfilename='%s-%i'%(newfilename,count)
					if l_filename==newfilename:
						continue #if the filenames are the same they would cmp the same and be deleted. (ex. when renameformat="%(fullname)s")
					if os.path.exists(newfilename):
						if fcmp(l_filename, newfilename):
							os.unlink(l_filename)
							reninfo=' (dupe of %s removed)'%newfilename
							break
					else:
						os.rename(l_filename, newfilename)
						reninfo=' (renamed to %s)'%newfilename
						break
			perror('%s : %s%s'%(perhaps_showpath(l_filename),a,reninfo))
			return -2
		stats.ok=stats.ok+1
		if config.list&LISTOK:
			plistf(filename)
		if filesize>=0:
			pverbose('%s : OK (%i,%s)'%(perhaps_showpath(filename),fs,filecrc))
		else:
			pverbose('%s : OK (%s)'%(perhaps_showpath(filename),filecrc))
	
	def make_chksumfile(self, file):
		self.wfile = file

#---------- md5 ----------

class MD5_MixIn:
	def do_test_file(self, filename, filecrc):
		c=getfilemd5(filename)[0]
		if c!=string.lower(filecrc):
			return c

class MD5(ChksumType, MD5_MixIn):
	def auto_chksumfile_match(file):
		return re.match(r'[0-9a-fA-F]{32} [ *].+', file.peekline(4096)) is not None
	auto_chksumfile_match=staticmethod(auto_chksumfile_match)

	def auto_filename_match(filename):
		return string.find(filename,'md5')>=0
	auto_filename_match = staticmethod(auto_filename_match)
	
	_md5rem=re.compile(r'([0-9a-fA-F]{32}) ([ *])(.+)$')
	def do_test_chksumline(self, l):
		x=self._md5rem.match(chomp(l))
		if not x: return -1
		if x.group(2)==' ':
			perror('warning: file %s tested in textmode' %x.group(3))
		self.test_file(x.group(3),x.group(1))
	
	def make_std_filename(filename):
		return filename+'.md5'
	make_std_filename = staticmethod(make_std_filename)
	
	def make_addfile(self, filename):
		self.wfile.write('%s *%s\n'%(getfilemd5(filename)[0],filename))

cftypes['md5']=MD5


#---------- bsdmd5 ----------

class BSDMD5(ChksumType, MD5_MixIn):
	def auto_chksumfile_match(file):
		return re.match(r'MD5 \(.+\) = [0-9a-fA-F]{32}'+'[\r\n]*$', file.peekline(4096)) is not None
	auto_chksumfile_match=staticmethod(auto_chksumfile_match)

	def auto_filename_match(filename):
		return string.lower(filename)=='md5' #####hm, make this have priority over md5sum type??
	auto_filename_match = staticmethod(auto_filename_match)
	
	_bsdmd5rem=re.compile(r'MD5 \((.+)\) = ([0-9a-fA-F]{32})$')
	def do_test_chksumline(self, l):
		x=self._bsdmd5rem.match(chomp(l))
		if not x: return -1
		self.test_file(x.group(1),x.group(2))
	
	def make_std_filename(filename):
		return filename+'.md5'
	make_std_filename = staticmethod(make_std_filename)
	
	def make_addfile(self, filename):
		self.wfile.write('MD5 (%s) = %s\n'%(filename, getfilemd5(filename)[0]))

cftypes['bsdmd5']=BSDMD5


#---------- par ----------

def ver2str(v):
	vers=[]
	while v or len(vers)<3:
		vers.insert(0, str(v&0xFF))
		v = v >> 8
	return string.join(vers, '.')

import struct
try: #support for 64bit ints in struct module was only added in python 2.2
	struct.calcsize('< Q') 
except struct.error:
	class Struct:
		_calcsize = struct.calcsize
		_unpack = struct.unpack
		def calcsize(self, fmt):
			return self._calcsize(re.sub('Q', 'II', fmt))
		def unpack(self, fmt, data):
			ofmt = re.sub('Q', 'II', fmt)
			unpacked = list(self._unpack(ofmt, data))
			ret = []
			for f in fmt.split(' '):
				if f=='Q':
					ret.append(long(unpacked.pop(0))+long(unpacked.pop(0))*2**32L)
				elif f=='<':pass
				else:
					ret.append(unpacked.pop(0))
			return tuple(ret)
	struct = Struct()

class PAR(ChksumType, MD5_MixIn):
	def auto_chksumfile_match(file):
		return file.peek(8)=='PAR\0\0\0\0\0'
	auto_chksumfile_match=staticmethod(auto_chksumfile_match)

	def do_test_chksumfile(self, file):
		def prog2str(v):
			return {0x01: 'Mirror', 0x02: 'PAR', 0x03:'SmartPar', 0xFF:'FSRaid'}.get(v, 'unknown(%x)'%v)
		par_header_fmt = '< 8s I I 16s 16s Q Q Q Q Q Q'
		par_entry_fmt = '< Q Q Q 16s 16s'
		par_entry_fmtsize = struct.calcsize(par_entry_fmt)

		d = file.read(struct.calcsize(par_header_fmt))
		magic, version, client, control_hash, set_hash, vol_number, num_files, file_list_ofs, file_list_size, data_ofs, data_size = struct.unpack(par_header_fmt, d)
		if config.docrcchecks:
			import md5
			control_md5 = md5.new()
			control_md5.update(d[0x20:])
			stats.bytesread=stats.bytesread+len(d)
		if version not in (0x00000900, 0x00010000): #ver 0.9 and 1.0 are the same, as far as we care.  Future versions (if any) may very likey have incompatible changes, so don't accept them either.
			raise EnvironmentError, (errno.EINVAL,"can't handle PAR version %s"%ver2str(version))

		pverbose('testing from %s (par v%s, created by %s v%s)'%(file.name,ver2str(version), prog2str(client>>24), ver2str(client&0xFFFFFF)))

		for i in range(0, num_files):
			d = file.read(par_entry_fmtsize)
			size, status, file_size, md5, md5_16k = struct.unpack(par_entry_fmt, d)
			if config.docrcchecks:
				control_md5.update(d)
			d = file.read(size - par_entry_fmtsize)
			filename = unicode(d, 'utf-16')
			if config.docrcchecks:
				control_md5.update(d)
				stats.bytesread=stats.bytesread+size
			c = "%02x"*16 % tuple(map(ord, md5))
			self.test_file(filename, c, file_size)

		if config.docrcchecks:
			while 1:
				d=file.read(65536)
				if not len(d):
					if control_md5.digest() != control_hash:
						raise EnvironmentError, (errno.EINVAL,"corrupt par file - bad control hash")
					break
				stats.bytesread=stats.bytesread+len(d)
				control_md5.update(d)

	#we don't support PAR in create mode, but add these methods so that we can get error messages that are probaby more user friendly.
	def auto_filename_match(filename):
		return filename[-3:]=='par'
	auto_filename_match = staticmethod(auto_filename_match)

	def make_std_filename(filename):
		return filename+'.par'
	make_std_filename = staticmethod(make_std_filename)

cftypes['par']=PAR


#---------- sfv ----------

class CRC_MixIn:
	def do_test_file(self, filename, filecrc):
		c=getfilecrc(filename)[0]
		if string.atol(c,16)!=string.atol(filecrc,16):
			return c
		
class SFV(ChksumType, CRC_MixIn):
	def auto_chksumfile_match(file):
		#(r';.*generated (by|using) (.* on|.*sfv|.*crc32)',re.IGNORECASE),
		return re.match(';|.+ [0-9a-fA-F]{8}[\n\r]*$', file.peekline(4096)) is not None
	auto_chksumfile_match=staticmethod(auto_chksumfile_match)

	def auto_filename_match(filename):
		return filename[-3:]=='sfv'
	auto_filename_match = staticmethod(auto_filename_match)
	
	def do_test_chksumfile_print_testingline(self, file):
		#override the default testing line to show first SFV comment line, if any
		comment = file.peekline()
		if comment[0]==';':
			comment = ', ' + comment[1:].strip()
			if len(comment)>102: #but limit the length in case its a really long one.
				comment = comment[:99]+'...'
		else:
			comment = ''
		pverbose('testing from %s (sfv%s)'%(file.name, comment))

	_sfvrem=re.compile(r'(.+) ([0-9a-fA-F]+)$')
	def do_test_chksumline(self, l):
		if l[0]==';': return
		x=self._sfvrem.match(chomp(l))
		if not x: return -1
		self.test_file(x.group(1),x.group(2))
	
	def make_std_filename(filename):
		return filename+'.sfv'
	make_std_filename = staticmethod(make_std_filename)
	
	def make_chksumfile(self, file):
		ChksumType.make_chksumfile(self, file)
		file.write('; Generated by cfv v%s on %s\n;\n'%(version,time.strftime('%Y-%m-%d at %H:%M.%S',time.gmtime(time.time()))))
		
	def make_addfile(self, filename):
		self.wfile.write('%s %s\n'%(filename,getfilecrc(filename)[0]))

cftypes['sfv']=SFV


#---------- csv ----------

class CSV(ChksumType, CRC_MixIn):
	def auto_chksumfile_match(file):
		return re.match('[^,]+,[0-9]+,[0-9a-fA-F]+,[\n\r]*$', file.peekline(4096)) is not None
	auto_chksumfile_match=staticmethod(auto_chksumfile_match)

	def auto_filename_match(filename):
		return filename[-3:]=='csv'
	auto_filename_match = staticmethod(auto_filename_match)
	
	_csvrem=re.compile(r'([^,]+),([0-9]+),([0-9a-fA-F]+),')
	def do_test_chksumline(self, l):
		x=self._csvrem.match(l)
		if not x: return -1
		self.test_file(x.group(1),x.group(3),int(x.group(2)))
	
	def make_std_filename(filename):
		return filename+'.csv'
	make_std_filename = staticmethod(make_std_filename)
	
	def make_addfile(self, filename):
		c,s=getfilecrc(filename)
		self.wfile.write('%s,%i,%s,\n'%(filename,s,c))

cftypes['csv']=CSV


#---------- csv with 4 fields ----------

class CSV4(ChksumType, CRC_MixIn):
	def auto_chksumfile_match(file):
		return re.match(r'[^,]+,[0-9]+,[0-9a-fA-F]+,[^,]*,', file.peekline(4096)) is not None
	auto_chksumfile_match=staticmethod(auto_chksumfile_match)

	_csv4rem=re.compile(r'([^,]+),([0-9]+),([0-9a-fA-F]+),([^,]*),')
	def do_test_chksumline(self, l):
		x=self._csv4rem.match(l)
		if not x: return -1
		self.test_file(os.path.join(fixpath(x.group(4)),x.group(1)),x.group(3),int(x.group(2))) #we need to fixpath before path.join since os.path.join looks for path.sep
	
	def make_std_filename(filename):
		return filename+'.csv'
	make_std_filename = staticmethod(make_std_filename)
	
	def make_addfile(self, filename):
		c,s=getfilecrc(filename)
		p=os.path.split(filename)
		self.wfile.write('%s,%i,%s,%s,\n'%(p[1],s,c,p[0]))

cftypes['csv4']=CSV4


#---------- csv with only 2 fields ----------

class CSV2(ChksumType):
	def auto_chksumfile_match(file):
		return re.match('[^,]+,[0-9]+,[\n\r]*$', file.peekline(4096)) is not None
	auto_chksumfile_match=staticmethod(auto_chksumfile_match)

	_csv2rem=re.compile(r'([^,]+),([0-9]+),')
	def do_test_chksumline(self, l):
		x=self._csv2rem.match(l)
		if not x: return -1
		self.test_file(x.group(1),None,int(x.group(2)))
	
	def make_std_filename(filename):
		return filename+'.csv'
	make_std_filename = staticmethod(make_std_filename)
	
	def make_addfile(self, filename):
		if filename=='':
			s=getfilecrc(filename)[1]#no way to get size of stdin other than to read it
		else:
			s=os.path.getsize(filename)
		self.wfile.write('%s,%i,\n'%(filename,s))

cftypes['csv2']=CSV2


#---------- generic ----------

def perhaps_showpath(file):
	if config.showpaths==1 or (config.showpaths==2 and config.recursive):
		if config.showpathsabsolute:
			dir=curdir
		else:
			dir=reldir[-1]
		return os.path.join(dir,file)
	return file

ncdc={}
def nocase_dirfiles(dir,match):
	"return list of filenames in dir whose string.lower() equals match"
	dir=os.path.normpath(os.path.join(curdir,dir))
	if not ncdc.has_key(dir):
		d={}
		ncdc[dir]=d
		for a in os.listdir(dir):
			l=string.lower(a)
			if d.has_key(l):
				d[l].append(a)
			else:
				d[l]=[a]
	else:
		d=ncdc[dir]
	if d.has_key(match):
		return d[match]
	return []

def path_split(filename):
	"returns a list of components of filename"
	head=filename
	parts=[]
	while 1:
		head,tail=os.path.split(head)
		if len(tail):
			parts.insert(0,tail)
		else:
			if len(head):
				parts.insert(0,head)
			break
	return parts

def nocase_findfile(filename):
	cur="."
	parts=map(string.lower, path_split(filename))
	#print 'nocase_findfile:',filename,parts,len(parts)
	for i in range(0,len(parts)):
		p=parts[i]
		#matches=filter(lambda f,p=p: string.lower(f)==p,dircache.listdir(cur)) #too slooow, even with dircache (though not as slow as without it ;)
		matches=nocase_dirfiles(cur,p) #nice and speedy :)
		#print 'i:',i,' cur:',cur,' p:',p,' matches:',matches
		if i==len(parts)-1:#if we are on the last part of the path, we want to match a file
			matches=filter(lambda f,cur=cur: os.path.isfile(os.path.join(cur,f)), matches)
		else:#otherwise, we want to match a dir
			matches=filter(lambda f,cur=cur: os.path.isdir(os.path.join(cur,f)), matches)
		if len(matches)==0:
			raise IOError, (errno.ENOENT,os.strerror(errno.ENOENT))
		if len(matches)>1:
			raise IOError, (errno.EEXIST,"More than one name matches %s"%os.path.join(cur,p))
		if cur=='.':
			cur=matches[0] #don't put the ./ on the front of the name
		else:
			cur=os.path.join(cur,matches[0])
	if filename != cur:
		stats.diffcase = stats.diffcase + 1
	return cur
	
def strippath(filename, num='a'):
	if num=='a':#split all the path off
		return os.path.split(filename)[1]
	if num=='n':#split none of the path
		return filename

	if re.match(r"[a-z]:\\",filename[0:3],re.I): #we can't use os.path.splitdrive, since we want to get rid of it even if we are not on a dos system.
		filename=filename[3:]
	if filename[0]==os.sep:
		filename=filename[1:]
	
	if num==0:#only split drive letter/root slash off
		return filename

	parts = path_split(filename)
	if len(parts) <= num:
		return parts[-1]
	return os.path.join(*parts[num:])

def fixpath(filename):
	if config.fixpaths:
		return re.sub('['+re.escape(config.fixpaths)+']', os.sep, filename)
	return filename


def test(filename,testfiles):
	try:
		file=doopen(filename,'r')
		for typename,cftype in cftypes.items():
			if cftype.auto_chksumfile_match(file):
				cf = cftype(testfiles)
				if config.showunverified and filename in stats.unverifiedfiles[-1]:#we can't expect the checksum file itself to be checksummed
					stats.unverifiedfiles[-1].remove(filename)
				cf.test_chksumfile(file, filename)
				return
	except EnvironmentError, a:
		stats.cferror=stats.cferror+1
		perror('%s : %s (CF)'%(perhaps_showpath(filename),a[1]))
		return -1
	perror("I don't recognize the type of %s"%filename)
	stats.cferror=stats.cferror+1

def make(cftype,ifilename,testfiles):
	file=None
	if ifilename:
		filename=ifilename
	else:
		filename=cftype.make_std_filename(os.path.basename(curdir))
		if config.gzip==1 and filename[-3:]!='.gz': #if user does -zz, perhaps they want to force the filename to be kept?
			filename=filename+'.gz'
	if not hasattr(cftype, "make_addfile"):
		perror("%s : %s not supported in create mode"%(filename, cftype.__name__.lower()))
		stats.cferror=stats.cferror+1
		return
	if os.path.exists(filename):
		perror("%s already exists"%perhaps_showpath(filename))
		stats.cferror=stats.cferror+1
		file=IOError #just need some special value to indicate a cferror so that recursive mode still continues to work, IOError seems like a good choice ;)
	if not testfiles or not len(testfiles):
		tfauto=1
		testfiles=os.listdir('.')
		if config.dirsort:
			testfiles.sort()
	else:
		tfauto=0
	testdirs=[]
	
	i=0
	while i<len(testfiles):
		f=testfiles[i]
		i=i+1
		if not tfauto and f=='-':
			f=''
		elif not os.path.isfile(f):
			if config.recursive and os.path.isdir(f):
				if config.recursive==1:
					testdirs.append(f)
				elif config.recursive==2:
					try:
						rfiles=os.listdir(f)
						if config.dirsort:
							rfiles.sort()
						testfiles[:i]=map(lambda x,p=f: os.path.join(p,x), rfiles)
						i=0
					except EnvironmentError, a:
						perror('%s%s : %s'%(f, os.sep, a[1]))
						stats.ferror=stats.ferror+1
				continue
			if tfauto:#if user isn't specifying files, don't even try to add dirs and stuff, and don't print errors about it.
				continue
		stats.num=stats.num+1
		if file==IOError:
			continue
		if file==None:
			file=doopen(filename,'w')
			cf = cftype(None)
			cf.make_chksumfile(file)
			dof=cf.make_addfile
		try:
			dof(f)
		except EnvironmentError, a:
			if a[0]==errno.ENOENT:
				stats.notfound=stats.notfound+1
			else:
				stats.ferror=stats.ferror+1
			perror('%s : %s'%(f,a[1]))
			continue
		stats.ok=stats.ok+1
	if file and file!=IOError:
		file.close() #### should this call cf.close or something instead?
	
	for f in testdirs:
		try:
			chdir(f)
		except EnvironmentError, a:
			perror('%s%s : %s'%(f, os.sep, a[1]))
			stats.ferror=stats.ferror+1
		else:
			make(cftype,ifilename,None)
			cdup()

def start_test_dir(args):
	if not config.showunverified:
		return
	if len(args):
		filelist=args[:] #make a copy so we don't modify the args list (teehee)
	else:
		filelist=os.listdir('.')
	#dir=curdir
	#stats.unverifiedfiles[dir]=map(lambda d: os.path.join(dir,d),filelist))
	#stats.unverifiedfiles[dir]=filelist
	stats.unverifiedfiles.append(filelist)
def finish_test_dir():
	if not config.showunverified:
		return
	unverified=stats.unverifiedfiles.pop()
	for file in unverified:
		if os.path.isfile(file):
			if config.list&LISTUNVERIFIED:
				plistf(file)
			perror('%s : not verified'%perhaps_showpath(file))
			stats.unverified=stats.unverified+1

atrem=re.compile(r'md5|\.(csv|sfv|par)(\.gz)?$',re.IGNORECASE)#md5sum files have no standard extension, so just search for files with md5 in the name anywhere, and let the test func see if it really is one.
def autotest(args):
	start_test_dir(args)
	for a in os.listdir('.'):
		if config.recursive and os.path.isdir(a):
			try:
				chdir(a)
			except EnvironmentError, e:
				perror('%s%s : %s'%(a, os.sep, e[1]))
				stats.ferror=stats.ferror+1
			else:
				autotest(args)
				cdup()
		if atrem.search(a):
			test(a,args)
	finish_test_dir()

def printusage(err=0):
	perror('Usage: cfv [opts] [-p dir] [-T|-C] [-t type] [-f file] [files...]')
	perror('  -r       recursive mode 1 (make seperate chksum files for each dir)')
	perror('  -rr      recursive mode 2 (make a single file with deep listing in it)')
	perror('  -R       not recursive (default)')
	perror('  -T       test mode (default)')
	perror('  -C       create mode')
	perror('  -t <t>   set type to <t> (%s, or auto(default))'%string.join(cftypes.keys(),', ')) #reduce(lambda a,b: a+b+', ',types.keys(),''))
	perror('  -f <f>   use <f> as list file')
	perror('  -m       check only for missing files (don\'t compare checksums)')
	perror('  -M       check checksums (default)')
	perror('  -n       rename bad files')
	perror('  -N       don\'t rename bad files (default)')
	perror('  -p <d>   change to directory <d> before doing anything')
	perror('  -i       ignore case')
	perror('  -I       don\'t ignore case (default)')
	perror('  -u       show unverified files')
	perror('  -U       don\'t show unverified files (default)')
	perror('  -v       verbose')
	perror('  -V       not verbose (default)')
	perror('  -VV      don\'t print status line at end either')
	perror('  -q       quiet mode.  check exit code for success.')
	perror('  -Q       mostly quiet mode.  only prints status lines.')
	perror('  -zz      force making gzipped files, even if not ending in .gz')
	perror('  -z       make gzipped files in auto create mode')
	perror('  -Z       don\'t create gzipped files automatically. (default)')
	perror('  -ZZ      never use gzip, even if file ends in .gz')
	perror(' --list=<l> raw list files of type <l> (%s)'%string.join(LISTARGS.keys(),', '))
	perror(' --list0=<l> same as list, but seperate files with nulls (useful for xargs -0)')
	perror(' --fixpaths=<s>  replace any chars in <s> with %s'%os.sep)
	perror(' --strippaths=VAL  strip leading components from file names.')
	perror(' --showpaths=<p> show full paths (none/auto/yes-absolute/relative)')
	perror(' --renameformat=<f> format string to use with -n option')
	perror(' --help/-h show help')
	perror(' --version show cfv and module versions')
	sys.exit(err)
def printhelp():
	perror('cfv v%s - Copyright (C) 2000-2002 Matthew Mueller - GPL license'%version)
	printusage()

stats=Stats()
config=Config()


def main(argv):
	manual=0
	mode=0
	typename='auto'

	try:
		optlist, args = getopt.getopt(argv, 'rRTCt:f:mMnNp:uUiIvVzZqQh?', ['list=', 'list0=', 'fixpaths=', 'strippaths=', 'showpaths=','renameformat=','help','version'])
	except getopt.error, a:
		perror("cfv: %s"%a)
		printusage(1)

	try:
		if config.cmdlineglob=='y' or (config.cmdlineglob=='a' and os.name in ('os2', 'nt', 'dos')):
			from glob import glob
			globbed = []
			for arg in args:
				if '*' in arg or '?' in arg or '[' in arg:
					g = glob(arg)
					if not g:
						raise CFVValueError, 'no matches for %s'%arg
					globbed.extend(g)
				else:
					globbed.append(arg)
			args = globbed
			
		if config.cmdlinesort:
			args.sort()

		prevopt=''
		for o,a in optlist:
			if o=='-T':
				mode=0
			elif o=='-C':
				mode=1
			elif o=='-t':
				if not a in ['auto']+cftypes.keys():
					raise CFVValueError, 'type %s not recognized'%a
				typename=a
			elif o=='-f':
				manual=1 #filename selected manually, don't try to autodetect
				filename=a
				if mode==0:
					if config.ignorecase:
						args=map(string.lower,args) #lowercase it all now, so we don't have to keep doing it over and over in the testfile
					start_test_dir(args)
					if typename=='auto':
						test(a,args)
					else:
						cf = cftypes[typename](args)
						cf.test_chksumfile(None,filename)
					finish_test_dir()
				else:
					if typename!='auto':
						make(cftypes[typename],a,args)
					else:
						ok=0
						testa=''
						if config.gzip>=0 and a[-3:]=='.gz':
								testa=a[:-3]
						for cftype in cftypes.values():
							if cftype.auto_filename_match(a) or (testa and cftype.auto_filename_match(testa)):
								make(cftype,a,args)
								ok=1
								break;
						if not ok:
							raise CFVValueError, 'specify a filetype with -t, or use standard extension'
			elif o=='-U':
				config.showunverified=0
			elif o=='-u':
				config.showunverified=1
			elif o=='-I':
				config.ignorecase=0
			elif o=='-i':
				config.ignorecase=1
			elif o=='-n':
				config.rename=1
			elif o=='-N':
				config.rename=0
			elif o=='--renameformat':
				config.setx('renameformat', a)
			elif o=='-m':
				config.docrcchecks=0
			elif o=='-M':
				config.docrcchecks=1
			elif o=='-p':
				chdir(a)
			elif o=='-r':
				if prevopt=='-r':
					config.recursive=2
				else:
					config.recursive=1
			elif o=='-R':
				config.recursive=0
			elif o=='-v':
				config.setx('verbose', 'v')
			elif o=='-V':
				if prevopt=='-V':
					config.setx('verbose', 'VV')
				else:
					config.setx('verbose', 'V')
			elif o=='-q':
				config.setx('verbose', 'q')
			elif o=='-Q':
				config.setx('verbose', 'Q')
			elif o=='-z':
				if prevopt=='-z':
					config.gzip=2
				else:
					config.gzip=1
			elif o=='-Z':
				if prevopt=='-Z':
					config.gzip=-1
				else:
					config.gzip=0
			elif o=='--list' or o=='--list0':
				if a not in LISTARGS.keys():
					raise CFVValueError, 'list arg must be one of '+`LISTARGS.keys()`
				config.list=LISTARGS[a]
				config.listsep = o=='--list0' and '\0' or '\n'
				if config.list==LISTUNVERIFIED:
					config.showunverified=1
				global p
				p=_perr #redirect all messages to stderr so only the list gets on stdout
			elif o=='--showpaths':
				config.setx('showpaths', a)
			elif o=='--strippaths':
				config.setx('strippaths', a)
			elif o=='--fixpaths':
				config.fixpaths=a
			elif o=='-h' or o=='-?' or o=='--help':
				printhelp()
			elif o=='--version':
				print 'cfv %s'%version
				try:
					if not nommap: print '+mmap'
				except NameError: pass
				try: print 'fchksum %s'%fchksum.version()
				except NameError: pass
				print 'python %08x-%s'%(sys.hexversion,sys.platform)
				sys.exit(0)
			prevopt=o
	except CFVValueError, e:
		perror('cfv: %s'%e)
		sys.exit(1)

	if not manual:
		if mode==0:
			if config.ignorecase:
				args=map(string.lower,args) #lowercase it all now, so we don't have to keep doing it over and over in the testfile
			autotest(args)
		else:
			if typename=='auto':
				typename=config.defaulttype
			make(cftypes[typename],None,args)
	
	#only print total stats if more than one checksum file has been checked. (or if none have)
	#We must also print stats here if there are unverified files or checksum file errors, since those conditions occur outside of the cf_stats section.
	if stats.subcount != 1 or stats.unverified or stats.cferror:
		stats.print_stats()

	sys.exit((stats.badcrc and 2) | (stats.badsize and 4) | (stats.notfound and 8) | (stats.ferror and 16) | (stats.unverified and 32) | (stats.cferror and 64))

if __name__=='__main__':
	main(sys.argv[1:])
