changelog shortlog tags branches files raw gz bz2 help

Mercurial > hg > ventivac / changeset: add -q to vacput, for only partially writing append-only files to venti

changeset 74: f09fa771561e
parent 73: ced42c12c2d2
child 75: f79e92c9fe4e
author: Mechiel Lukkien <mechiel@ueber.net>
date: Tue, 17 Jul 2007 22:32:49 +0200
files: appl/cmd/vacput.b appl/lib/vac.b man/1/vacget module/vac.m
description: add -q to vacput, for only partially writing append-only files to venti
     1.1--- a/appl/cmd/vacput.b	Tue Jul 17 16:34:20 2007 +0200
     1.2+++ b/appl/cmd/vacput.b	Tue Jul 17 22:32:49 2007 +0200
     1.3@@ -30,7 +30,7 @@
     1.4 };
     1.5 
     1.6 addr := "net!$venti!venti";
     1.7-Dflag, vflag, rflag: int;
     1.8+Dflag, vflag, qflag, rflag: int;
     1.9 blocksize := vac->Dsize;
    1.10 session: ref Session;
    1.11 name := "vac";
    1.12@@ -64,7 +64,7 @@
    1.13 	blockmax = 32*1024;
    1.14 
    1.15 	arg->init(args);
    1.16-	arg->setusage(sprint("%s [-Drv] [-d base] [-a addr] [-b blocksize] [-n name] path ...", arg->progname()));
    1.17+	arg->setusage(sprint("%s [-Dqrv] [-d base] [-a addr] [-b blocksize] [-n name] path ...", arg->progname()));
    1.18 	while((c := arg->opt()) != 0)
    1.19 		case c {
    1.20 		'a' =>	addr = arg->earg();
    1.21@@ -85,6 +85,7 @@
    1.22 		'D' =>	Dflag++;
    1.23 			vac->dflag++;
    1.24 			rabin->debug++;
    1.25+		'q' =>	qflag++;
    1.26 		'r' =>	rflag++;
    1.27 		'v' =>	vflag++;
    1.28 		* =>	warn(sprint("bad option: -%c", c));
    1.29@@ -239,12 +240,30 @@
    1.30 			nde = vd.walk(dir.name);
    1.31 		if(nde != nil)
    1.32 			(e, nil) = vd.open(nde);
    1.33-		# xxx use qid version
    1.34-		if(e != nil && (nde.mtime != dir.mtime || e.size != dir.length || nde.qid != dir.qid.path))
    1.35+		f: ref File;
    1.36+		offset := big 0;
    1.37+		if(e != nil && qflag
    1.38+			&& (nde.mode&Vac->Modeappend)
    1.39+			&& (dir.mode&Sys->DMAPPEND)
    1.40+			&& nde.mtime < dir.mtime
    1.41+			&& e.size < dir.length
    1.42+			&& nde.qid == dir.qid.path
    1.43+			&& nde.mcount < dir.qid.vers)
    1.44+		{ 
    1.45+			f = File.mkstate(session, e, rflag);
    1.46+			offset = f.size;
    1.47+		} else if(f == nil && e != nil
    1.48+			&& (nde.mtime != dir.mtime
    1.49+			|| e.size != dir.length
    1.50+			|| nde.qid != dir.qid.path
    1.51+			|| nde.mcount != dir.qid.vers))
    1.52 			e = nil;
    1.53-		if(e == nil) {
    1.54+		if(f == nil && e == nil)
    1.55+			f = File.new(session, Datatype, blocksize, rflag);
    1.56+		
    1.57+		if(f != nil) {
    1.58 say("writepath: file has changed or is new, writing it");
    1.59-			e = writefile(path, fd);
    1.60+			e = writefile(path, fd, f, offset);
    1.61 			if(e == nil)
    1.62 				error(sprint("error flushing filesink for %s: %r", path));
    1.63 		}
    1.64@@ -270,12 +289,13 @@
    1.65 say("writepath done");
    1.66 }
    1.67 
    1.68-writefile(path: string, fd: ref Sys->FD): ref Entry
    1.69+writefile(path: string, fd: ref Sys->FD, f: ref File, offset: big): ref Entry
    1.70 {
    1.71 	bio := bufio->fopen(fd, bufio->OREAD);
    1.72 	if(bio == nil)
    1.73 		error(sprint("bufio opening %s: %r", path));
    1.74-	say(sprint("bufio opened path %s", path));
    1.75+	say(sprint("bufio opened path %s, offset=%bd", path, offset));
    1.76+	bio.seek(offset, Bufio->SEEKSTART);
    1.77 
    1.78 	rfile: ref Rfile;
    1.79 	if(rflag) {
    1.80@@ -285,7 +305,6 @@
    1.81 			error(sprint("rabin open %s: %s", path, err));
    1.82 	}
    1.83 
    1.84-	f := File.new(session, Datatype, blocksize, rflag);
    1.85 	for(;;) {
    1.86 		if(rflag) {
    1.87 			(d, nil, err) := rfile.read();
     2.1--- a/appl/lib/vac.b	Tue Jul 17 16:34:20 2007 +0200
     2.2+++ b/appl/lib/vac.b	Tue Jul 17 22:32:49 2007 +0200
     2.3@@ -208,7 +208,7 @@
     2.4 		mode |= Modedir;
     2.5 	if(d.mode&sys->DMTMP)
     2.6 		mode |= Modetemp;
     2.7-	return ref Direntry(9, d.name, 0, 0, 0, 0, d.qid.path, d.uid, d.gid, d.muid, d.mtime, 0, 0, atime, mode, d.mode);
     2.8+	return ref Direntry(9, d.name, 0, 0, 0, 0, d.qid.path, d.uid, d.gid, d.muid, d.mtime, d.qid.vers, 0, atime, mode, d.mode);
     2.9 }
    2.10 
    2.11 Direntry.mkdir(de: self ref Direntry): ref Sys->Dir
    2.12@@ -513,6 +513,45 @@
    2.13 	return e;
    2.14 }
    2.15 
    2.16+File.mkstate(session: ref Venti->Session, e: ref Entry, rabin: int): ref File
    2.17+{
    2.18+	s := e.score;
    2.19+	dsize := e.dsize;
    2.20+	if(dsize == 0)
    2.21+		dsize = Dsize;	# xxx not clean
    2.22+	if(e.depth == 0) {
    2.23+		p := Page.new(dsize, rabin);
    2.24+		return ref File(array[1] of {p}, Datatype, dsize, big 0, session, rabin);
    2.25+	}
    2.26+	f := ref File(array[e.depth] of ref Page, Datatype, dsize, big 0, session, rabin);
    2.27+say(sprint("mkstate, depth=%d", e.depth));
    2.28+	for(i := 0; i < e.depth; i++) {
    2.29+say(sprint("mkstate: reading score=%s", e.score.text()));
    2.30+		d := session.read(s, Pointertype0+e.depth-1-i, Venti->Maxlumpsize);
    2.31+		if(d == nil)
    2.32+			return nil;
    2.33+		p := Page.new(dsize, rabin);
    2.34+		p.d[:] = d;
    2.35+		p.o = len d-p.esize;
    2.36+		s = Score(p.d[p.o:p.o+Scoresize]);
    2.37+		f.p[i] = p;
    2.38+	}
    2.39+	if(rabin) {
    2.40+		lp := f.p[len f.p-1];
    2.41+		bsize := g64(lp.d, lp.o+Scoresize);
    2.42+		f.size = e.size-bsize;
    2.43+		for(i = 0; i < len f.p; i++)
    2.44+			f.p[i].treesize -= bsize;
    2.45+	} else {
    2.46+		ls := e.size % big e.dsize;
    2.47+		if(ls == big 0)
    2.48+			ls = big e.dsize;
    2.49+		f.size = e.size-ls;
    2.50+	}
    2.51+say(sprint("mkstate: have file, size=%bd", f.size));
    2.52+	return f;
    2.53+}
    2.54+
    2.55 
    2.56 Sink.new(s: ref Venti->Session, dsize: int): ref Sink
    2.57 {
     3.1--- a/man/1/vacget	Tue Jul 17 16:34:20 2007 +0200
     3.2+++ b/man/1/vacget	Tue Jul 17 22:32:49 2007 +0200
     3.3@@ -39,12 +39,18 @@
     3.4 .B DMTMP
     3.5 bit set, are skipped.  With
     3.6 .B -d
     3.7-a base archive can be specified.  Files that have not changed are assumed to be present in venti and included in the new archive without writing their contents to venti.
     3.8+a base archive can be specified.  Files that have not changed are assumed to be present in venti and included in the new archive without writing their contents to venti.  Specify
     3.9+.B -q
    3.10+to allow vacput to only write new data in append-only files to venti.
    3.11 .TP
    3.12 .BI -d " tag:score"
    3.13 The archive denoted by
    3.14 .I tag:score
    3.15 is used as base archive while writing.  Only the contents of files that have been modified are written to venti before inclusion in the new archive.  Only for vacput.
    3.16+.TP
    3.17+.B -q
    3.18+Use heuristics (append-only mode, qid path, qid version, modification time and file size) to determine whether a part of the file is already in venti and does not have to be written again.  Only for vacput and in conjunction with
    3.19+.BR -d .
    3.20 .B -D
    3.21 Print debug messages.
    3.22 .TP
     4.1--- a/module/vac.m	Tue Jul 17 16:34:20 2007 +0200
     4.2+++ b/module/vac.m	Tue Jul 17 22:32:49 2007 +0200
     4.3@@ -112,7 +112,7 @@
     4.4 		data:	fn(p: self ref Page): array of byte;
     4.5 	};
     4.6 
     4.7-	# hash tree file
     4.8+	# for writing a hash tree file
     4.9 	File: adt {
    4.10 		p:	array of ref Page;
    4.11 		dtype, dsize:	int;
    4.12@@ -123,9 +123,10 @@
    4.13 		new:	fn(s: ref Venti->Session, dtype, dsize, rabin: int): ref File;
    4.14 		write:	fn(f: self ref File, d: array of byte): int;
    4.15 		finish:	fn(f: self ref File): ref Entry;
    4.16+		mkstate:	fn(session: ref Venti->Session, e: ref Entry, rabin: int): ref File;
    4.17 	};
    4.18 
    4.19-	# for writing venti directories
    4.20+	# for writing venti directories (entries)
    4.21 	Sink: adt {
    4.22 		f:	ref File;
    4.23 		d:	array of byte;