Merge pull request #25 from rfjakob/master

Merge bash tests into Perl infrastructure

Makefile.am

@@ -14,3 +14,11 @@ MAINTAINERCLEANFILES = aclocal.m4
 
 
 ACLOCAL_AMFLAGS = -I m4
+
+.PHONY: test
+test:
+	perl -MTest::Harness -e '$$Test::Harness::verbose=0; runtests @ARGV;' tests/*.pl
+
+.PHONY: test-verbose
+test-verbose:
+	perl -MTest::Harness -e '$$Test::Harness::verbose=1; runtests @ARGV;' tests/*.pl

README.md

@@ -31,6 +31,20 @@ depends upon community interest.  In order to make it easier for anyone to
 contribute, it is moving a new home on Github.  So if you're interested in
 EncFS, please dive in!
 
+EncFS still has a few unique features that may be interesing to you:
+
+*   `--reverse` mode: Provides an encrypted view of an unencrypted folder.
+    This enables encrypted remote backups using standard tools like
+    rsync.
+*   EncFS is typically faster than ecryptfs for stat()-heavy workloads
+    when the backing device is a classical hard disk.
+    This is because ecryptfs has to to read each file header to determine
+    the file size - EncFS does not. This is one additional seek for each
+    stat. On SSDs that have virtually no seek time, that difference may
+    disappear.
+*   EncFS works on network file systems (NFS, CIFS...), while ecryptfs
+    is known to still have [problems][1].
+
 ## GitHub Transition
 
 GitHub hosting for EncFS is a work in progress.  See also the original, and
@@ -46,3 +60,5 @@ work, some of which may be back-ported to the master branch when it is stable. T
 dev branch is not stable, and there is no guarantee of backward compatibility
 between changes.
 
+
+[1]: https://bugs.launchpad.net/ecryptfs/+bug/277578

encfs/Makefile.am

@@ -141,9 +141,3 @@ encfs-man.html: encfs.pod
 	@POD2HTML@ encfs.pod > $@
 endif
 
-tests:
-	perl -MTest::Harness -e '$$Test::Harness::verbose=0; runtests @ARGV;' *.t
-
-tests-verbose:
-	perl -MTest::Harness -e '$$Test::Harness::verbose=1; runtests @ARGV;' *.t
-

tests/default.xml

@@ -1,37 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
-<!DOCTYPE boost_serialization>
-<boost_serialization signature="serialization::archive" version="10">
-<cfg class_id="0" tracking_level="0" version="20">
-	<version>20100713</version>
-	<creator>EncFS 1.7.5</creator>
-	<cipherAlg class_id="1" tracking_level="0" version="0">
-		<name>ssl/aes</name>
-		<major>3</major>
-		<minor>0</minor>
-	</cipherAlg>
-	<nameAlg>
-		<name>nameio/block</name>
-		<major>4</major>
-		<minor>0</minor>
-	</nameAlg>
-	<keySize>192</keySize>
-	<blockSize>1024</blockSize>
-	<uniqueIV>1</uniqueIV>
-	<chainedNameIV>1</chainedNameIV>
-	<externalIVChaining>0</externalIVChaining>
-	<blockMACBytes>0</blockMACBytes>
-	<blockMACRandBytes>0</blockMACRandBytes>
-	<allowHoles>1</allowHoles>
-	<encodedKeySize>44</encodedKeySize>
-	<encodedKeyData>
-AL2iRMLYyuf7RXcFCK0CcegQ/+enbaaqTq1OY3WULMLpcz0yvo8K4KtVw7c=
-	</encodedKeyData>
-	<saltLen>20</saltLen>
-	<saltData>
-RVCTqJyxxp7ZN0AsLJhgpvSuUB8=
-	</saltData>
-	<kdfIterations>170562</kdfIterations>
-	<desiredKDFDuration>500</desiredKDFDuration>
-</cfg>
-</boost_serialization>
-

tests/mac.xml

@@ -1,37 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
-<!DOCTYPE boost_serialization>
-<boost_serialization signature="serialization::archive" version="10">
-<cfg class_id="0" tracking_level="0" version="20">
-	<version>20100713</version>
-	<creator>EncFS 1.7.5</creator>
-	<cipherAlg class_id="1" tracking_level="0" version="0">
-		<name>ssl/aes</name>
-		<major>3</major>
-		<minor>0</minor>
-	</cipherAlg>
-	<nameAlg>
-		<name>nameio/block</name>
-		<major>4</major>
-		<minor>0</minor>
-	</nameAlg>
-	<keySize>192</keySize>
-	<blockSize>1024</blockSize>
-	<uniqueIV>1</uniqueIV>
-	<chainedNameIV>1</chainedNameIV>
-	<externalIVChaining>0</externalIVChaining>
-	<blockMACBytes>8</blockMACBytes>
-	<blockMACRandBytes>0</blockMACRandBytes>
-	<allowHoles>1</allowHoles>
-	<encodedKeySize>44</encodedKeySize>
-	<encodedKeyData>
-AL2iRMLYyuf7RXcFCK0CcegQ/+enbaaqTq1OY3WULMLpcz0yvo8K4KtVw7c=
-	</encodedKeyData>
-	<saltLen>20</saltLen>
-	<saltData>
-RVCTqJyxxp7ZN0AsLJhgpvSuUB8=
-	</saltData>
-	<kdfIterations>170562</kdfIterations>
-	<desiredKDFDuration>500</desiredKDFDuration>
-</cfg>
-</boost_serialization>
-

tests/normal.pl

@@ -1,15 +1,18 @@
 #!/usr/bin/perl -w
 
+# Test EncFS normal and paranoid mode
+
 use Test::More qw( no_plan );
 use File::Path;
+use File::Copy;
+use File::Temp;
 use IO::Handle;
-use Digest::MD5;
+use Digest::MD5 qw(md5_hex);
 
 my $tempDir = $ENV{'TMPDIR'} || "/tmp";
 
-my $raw = "$tempDir/crypt-raw-$$";
+# run unit tests
-my $crypt = "$tempDir/crypt-$$";
+ok( system("./encfs/test 2> /dev/null") == 0, "unit tests");
-
 
 # test filesystem in standard config mode
 &runTests('standard');
@@ -17,12 +20,13 @@ my $crypt = "$tempDir/crypt-$$";
 # test in paranoia mode
 &runTests('paranoia');
 
-
+# Wrapper function - runs all tests in the specified mode
-
 sub runTests
 {
     my $mode = shift;
 
+    &newWorkingDir;
+
     my $hardlinks = 1;
     if($mode eq 'standard')
     {
@@ -31,6 +35,7 @@ sub runTests
     {
         &mount("--paranoia");
         $hardlinks = 0; # no hardlinks in paranoia mode
+        &corruption;
     } else
     {
         die "invalid test mode";
@@ -41,10 +46,77 @@ sub runTests
     &links($hardlinks);
     &truncate;
     &renames;
+    &internalModification;
+    &grow;
 
     &cleanup;
 }
 
+# Helper function
+# Create a new empty working directory
+sub newWorkingDir
+{
+    our $workingDir = mkdtemp("$tempDir/encfs-tests-XXXX")
+        || BAIL_OUT("Could not create temporary directory");
+
+    our $raw = "$workingDir/raw";
+    our $crypt = "$workingDir/crypt";
+}
+
+# Test Corruption
+# Modify the encrypted file and verify that the MAC check detects it
+sub corruption
+{
+    ok( open(OUT, "+> $crypt/corrupt") && print(OUT "12345678901234567890")
+        && close(OUT), "create corruption-test file" );
+
+
+    $e = encName("corrupt");
+    ok( open(OUT, ">> $raw/$e") && print(OUT "garbage") && close(OUT),
+        "corrupting raw file");
+
+    ok( open(IN, "< $crypt/corrupt"), "open corrupted file");
+    my $content;
+    $result = read(IN, $content, 20);
+    ok(! defined $result, "corrupted file with MAC returns read error: $!");
+}
+
+# Test internal modification
+# Create a file of fixed size and overwrite data at different offsets
+# (like a database would do)
+sub internalModification
+{
+    $ofile = "$workingDir/crypt-internal-$$";
+    qx(dd if=/dev/urandom of=$ofile bs=2k count=2 2> /dev/null);
+    ok(copy($ofile, "$crypt/internal"), "copying crypt-internal file");
+
+    open(my $out1, "+<", "$crypt/internal");
+    open(my $out2, "+<", $ofile);
+
+    @fhs = ($out1, $out2);
+
+    $ori = md5fh($out1);
+    $b = md5fh($out2);
+
+    ok( $ori eq $b, "random file md5 matches");
+
+    my @offsets = (10, 30, 1020, 1200);
+    foreach my $o (@offsets)
+    {
+        foreach my $fh(@fhs) {
+            seek($fh, $o, 0);
+            print($fh "garbagegarbagegarbagegarbagegarbage");
+        }
+        $a=md5fh($out1);
+        $b=md5fh($out2);
+        ok( ($a eq $b) && ($a ne $ori), "internal modification at $o");
+    }
+
+    close($out1);
+    close($out2);
+}
+
+# Test renames
 sub renames
 {
     ok( open(F, ">$crypt/orig-name") && close F, "create file for rename test");
@@ -73,6 +145,7 @@ sub renames
     is( (stat "$crypt/3rd-name")[9], $olderTime, "time unchanged by rename");
 }
 
+# Test truncate and grow
 sub truncate
 {
     # write to file, then truncate it
@@ -95,26 +168,25 @@ sub truncate
     print OUT "12345";
     is( -s "$crypt/trunc", 35, "truncated file size");
 
-    seek(OUT, 0, 0);
+    is( md5fh(*OUT), "5f170cc34b1944d75d86cc01496292df",
-    is( Digest::MD5->new->addfile(*OUT)->hexdigest, 
+        "content digest");
-        "5f170cc34b1944d75d86cc01496292df", "content digest");
 
     # try crossing block boundaries
     seek(OUT, 10000,0);
     print OUT "abcde";
-    seek(OUT, 0, 0);
+
-    is( Digest::MD5->new->addfile(*OUT)->hexdigest, 
+    is( md5fh(*OUT), "117a51c980b64dcd21df097d02206f98",
-        "117a51c980b64dcd21df097d02206f98", "content digest");
+        "content digest");
 
     # then truncate back to 35 chars
     truncate(OUT, 35);
-    seek(OUT, 0, 0);
+    is( md5fh(*OUT), "5f170cc34b1944d75d86cc01496292df",
-    is( Digest::MD5->new->addfile(*OUT)->hexdigest, 
+        "content digest");
-        "5f170cc34b1944d75d86cc01496292df", "content digest");
 
     close OUT;
 }
 
+# Test file creation and removal
 sub fileCreation
 {
     # create a file
@@ -122,8 +194,7 @@ sub fileCreation
     ok( -f "$crypt/df.txt", "file created" );
 
     # ensure there is an encrypted version.
-    my $c = qx(./encfsctl encode --extpass="echo test" $raw df.txt);
+    my $c = encName("df.txt");
-    chomp($c);
     cmp_ok( length($c), '>', 8, "encrypted name ok" );
     ok( -f "$raw/$c", "encrypted file created" );
 
@@ -136,6 +207,46 @@ sub fileCreation
     ok( ! -f "$raw/$c", "file removal" );
 }
 
+# Test file growth
+sub grow
+{
+    open(my $fh_a, "+>$crypt/grow");
+    open(my $fh_b, "+>$workingDir/grow");
+
+    my $d = "1234567"; # Length 7 so we are not aligned to the block size
+    my $len = 7;
+
+    my $old = "";
+    my $errs = 0;
+
+    my $i;
+    for($i=1; $i<1000; $i++)
+    {
+        print($fh_a $d);
+        print($fh_b $d);
+
+        my $a = md5fh($fh_a);
+        my $b = md5fh($fh_b);
+
+        my $size = $len * $i;
+
+        # md5sums must be identical but must have changed
+        if($a ne $b || $a eq $old)
+        {
+            $errs++;
+        }
+
+        $old = $a;
+    }
+
+    ok($errs == 0, "grow file by $len bytes, $i times");
+
+    close($fh_a);
+    close($fh_b);
+}
+
+# Helper function
+# Check a file's content
 sub checkContents
 {
     my ($file, $expected, $testName) = @_;
@@ -147,6 +258,28 @@ sub checkContents
     close IN;
 }
 
+# Helper function
+# Convert plain-text filename to encrypted filename
+sub encName
+{
+    my $plain = shift;
+    my $enc = qx(./encfs/encfsctl encode --extpass="echo test" $raw $plain);
+    chomp($enc);
+    return $enc;
+}
+
+# Helper function
+# Get the MD5 sum of the file open at the filehandle
+sub md5fh
+{
+    my $fh_orig = shift;
+    open(my $fh, "<&", $fh_orig); # Duplicate the file handle so the seek
+    seek($fh, 0, 0);              # does not affect the caller
+    return Digest::MD5->new->addfile($fh)->hexdigest;
+    close($fh);
+}
+
+# Test symlinks & hardlinks
 sub links
 {
     my $hardlinkTests = shift;
@@ -173,26 +306,27 @@ sub links
     };
 }
 
+# Test mount
+# Leaves the filesystem mounted - also used as a helper function
 sub mount
 {
     my $args = shift;
 
-    ok( ! -d $raw, "no existing dir");
+    # When these fail, the rest of the tests makes no sense
-    ok( ! -d $crypt, "no existing dir");
+    mkdir($raw) || BAIL_OUT("Could not create $raw: $!");
+    mkdir($crypt)  || BAIL_OUT("Could not create $crypt: $!");
 
-    mkdir $raw;
+    qx(./encfs/encfs --extpass="echo test" $args $raw $crypt);
-    ok( -d $raw, "created dir" );
-    mkdir $crypt;
-    ok( -d $crypt, "created dir" );
-
-    qx(./encfs --extpass="echo test" $args $raw $crypt);
 
     ok( -f "$raw/.encfs6.xml",  "created control file");
 }
 
+# Helper function
+# Unmount and delete mountpoint
 sub cleanup
 {
     my $fusermount = qx(which fusermount);
+    chomp($fusermount);
     if(-f $fusermount)
     {
         qx($fusermount -u "$crypt");
@@ -204,10 +338,7 @@ sub cleanup
     rmdir $crypt;
     ok( ! -d $crypt, "unmount ok, mount point removed");
 
-    if(-d $raw)
+    rmtree($workingDir);
-    {
+    ok( ! -d $workingDir, "working dir removed");
-        rmtree($raw);
-    }
-    ok( ! -d $raw, "encrypted directory removed");
 }
 

tests/reverse.pl

@@ -0,0 +1,66 @@
+#!/usr/bin/perl -w
+
+# Test EncFS --reverse mode
+
+use Test::More qw( no_plan );
+use File::Path;
+use File::Temp;
+
+my $tempDir = $ENV{'TMPDIR'} || "/tmp";
+
+# Helper function
+# Create a new empty working directory
+sub newWorkingDir
+{
+    our $workingDir = mkdtemp("$tempDir/encfs-reverse-tests-XXXX")
+        || BAIL_OUT("Could not create temporary directory");
+
+    our $plain = "$workingDir/plain";
+    mkdir($plain);
+    our $ciphertext = "$workingDir/ciphertext";
+    mkdir($ciphertext);
+    our $decrypted = "$workingDir/decrypted";
+    mkdir($decrypted);
+}
+
+# Helper function
+# Unmount and delete mountpoint
+sub cleanup
+{
+    system("fusermount -u $decrypted");
+    system("fusermount -u $ciphertext");
+
+    rmtree($workingDir);
+    ok( ! -d $workingDir, "working dir removed");
+}
+
+# Helper function
+# Mount encryption-decryption chain
+#
+# Directory structure: plain -[encrypt]-> ciphertext -[decrypt]-> decrypted
+sub mount
+{
+    my $r=system("encfs --extpass=\"echo test\" --standard $plain $ciphertext --reverse > /dev/null");
+    ok($r == 0, "mounted ciphertext file system");
+
+    $r=system("ENCFS6_CONFIG=$plain/.encfs6.xml encfs --extpass=\"echo test\" $ciphertext $decrypted");
+    ok($r == 0, "mounted decrypting file system");
+}
+
+# Copy a directory tree and verify that the decrypted data is identical
+sub copy_test
+{
+    ok(system("cp -a encfs $plain")==0, "copying files to plain");
+    ok(system("diff -r -q $plain $decrypted")==0, "decrypted files are identical");
+
+    ok(-f "$plain/encfs/encfs.cpp", "file exists");
+    unlink("$plain/encfs/encfs.cpp");
+    ok(! -f "$decrypted/encfs.cpp", "file deleted");
+}
+
+newWorkingDir();
+mount();
+
+copy_test();
+
+cleanup();

tests/run.sh

@@ -1,109 +0,0 @@
-#!/bin/bash
-
-# This is the wrapper script for all tests.
-# It calls the unit tests and all integration tests.
-#
-# Functions that are useful in more than one integration test
-# should be defined here.
-
-set -eu
-
-# Set up envrionment variables, cd to the correct directory
-function init {
-
-	# Make sure we are in the "tests" directory
-	TESTDIR=$(dirname $(realpath "$0"))
-	cd $TESTDIR
-
-	# Full path to encfs binary
-	ENCFS=$(realpath ../encfs/encfs)
-
-	# Directory for temporary files (scratch)
-	SCRATCH=$(realpath scratch)
-	LOWER=$SCRATCH/lower
-	UPPER=$SCRATCH/upper
-
-	# Test number counter
-	TESTNO=1
-
-	trap 'echo -e "***\e[31m test(s) FAILED\e[0m"' ERR
-}
-
-# Run all test_*.sh files
-function run_cli_tests {
-
-	cd $TESTDIR
-	for i in $(echo test_*.sh)
-	do
-		cleanup
-		mkdir $SCRATCH
-		echo "*** running $i"
-		source $i
-	done
-}
-
-# fresh_mount CONFIG
-#
-# Mount a fresh, empty encfs filesystem using the encfs config file
-# CONFIG as .encfs6.xml. The password must be set to "a".
-# The backing files will be at $LOWER, the mounted filesystem at $UPPER.
-function fresh_mount {
-	cd $TESTDIR
-	fusermount -q -u $UPPER 2> /dev/null || true
-	wait
-	rm -Rf $LOWER $UPPER
-	mkdir -p $LOWER $UPPER
-	touch $UPPER/not_yet_mounted
-
-	cp $TESTDIR/$1 $SCRATCH/lower/.encfs6.xml
-
-	echo a | $ENCFS -f -S -o nonempty $LOWER $UPPER 2> /dev/null &
-	while [ -e $UPPER/not_yet_mounted ]
-	do
-		sleep 0.1s
-	done
-}
-
-# Clean up scratch directory
-function cleanup {
-	test -d $SCRATCH || return 0
-
-	cd $SCRATCH/..
-	fusermount -q -u $UPPER 2> /dev/null || true
-	wait
-	rm -Rf $SCRATCH
-}
-
-# Get the plain MD5 sum of a file, without the filename that is output
-# by md5sum
-# md5sum foo:    5f47bbbd6db883f93f5d00fd05f149ff  foo
-# plain_md5 foo: 5f47bbbd6db883f93f5d00fd05f149ff
-function md5 {
-	OUTPUT=$(md5sum "$1") # Capture md5sum output
-	ARRAY=($OUTPUT)       # Split into array
-	echo ${ARRAY[0]}      # Return first element
-}
-
-# Indicate beginning of a test
-# Prints test number and title
-function test_begin {
-	echo -n "$TESTNO $1: "
-	let TESTNO++
-}
-
-# Indicate successful completion of a test
-function test_ok {
-	echo "OK"
-}
-
-init
-
-test_begin "Running unit tests"
-../encfs/test 2> /dev/null
-test_ok
-
-run_cli_tests
-
-echo -e "***\e[32m All tests OK\e[0m"
-
-cleanup

tests/test_corruption.sh

@@ -1,31 +0,0 @@
-fresh_mount default.xml
-cd $SCRATCH
-dd if=/dev/urandom of=foo bs=1M count=1 2> /dev/null
-A=$(md5 foo)
-cp foo $UPPER
-cd $UPPER
-test_begin "Reading file"
-B=$(md5 foo)
-test $A = $B
-test_ok
-
-test_begin "Reading corrupted file"
-echo DEADBEEF >> $LOWER/$(ls $LOWER)
-B=$(md5 foo)
-test $A != $B
-test_ok
-
-fresh_mount mac.xml
-cd $SCRATCH
-cp foo $UPPER
-cd $UPPER
-test_begin "Reading file with MAC"
-B=$(md5 foo)
-test $A = $B
-test_ok
-
-test_begin "Corruption with MAC returns IO error"
-echo DEADBEEF >> $LOWER/$(ls $LOWER)
-md5 foo 2>&1 | grep "Input/output error" > /dev/null
-test_ok
-

tests/test_file_expansion.sh

@@ -1,40 +0,0 @@
-fresh_mount default.xml
-cd $UPPER
-test_begin "Creating files of different sizes: "
-for i in `seq 0 50` `seq 1000 1050`
-do
-	OUTPUT=$(dd if=/dev/zero bs=$i count=1 2> /dev/null | tee $i | md5sum)
-	ARRAY=($OUTPUT)
-	A=${ARRAY[0]} # Remove filename
-	B=$(md5 $i)
-	test $A = $B
-done
-test_ok
-
-test_begin "Growing file"
-rm -f ../grow
-for i in `seq 0 300`
-do
-	echo -n "abcdefg" >> ../grow
-	echo -n "abcdefg" >> grow
-	
-	A=$(md5 ../grow)
-	B=$(md5 grow)
-	test "$A" = "$B"
-done
-test_ok
-
-test_begin "Internal modification"
-dd if=/dev/urandom of=../internal bs=1M count=2 2> /dev/null
-cp ../internal internal
-for i in 0 30 1020 1200
-do
-	dd if=/dev/zero of=../internal bs=1 count=1 skip=$i 2> /dev/null
-	dd if=/dev/zero of=internal bs=1 count=1 skip=$i 2> /dev/null
-
-	A=$(md5 ../internal)
-	B=$(md5 internal)
-	test "$A" = "$B"
-done
-test_ok
-