Merge pull request #34 from rfjakob/reverse-iv

reverse: Implement unique IV derived from the inode number

encfs/BlockFileIO.cpp

@@ -27,6 +27,8 @@
 
 #include "i18n.h"
 
+#include "FileUtils.h"
+
 template <typename Type>
 inline Type min(Type A, Type B) {
   return (B < A) ? B : A;
@@ -41,6 +43,7 @@ BlockFileIO::BlockFileIO(int blockSize, const FSConfigPtr &cfg)
     : _blockSize(blockSize), _allowHoles(cfg->config->allowHoles) {
   rAssert(_blockSize > 1);
   _cache.data = new unsigned char[_blockSize];
+  _noCache = cfg->opts->noCache;
 }
 
 BlockFileIO::~BlockFileIO() {
@@ -48,13 +51,27 @@ BlockFileIO::~BlockFileIO() {
   delete[] _cache.data;
 }
 
+/**
+ * Serve a read request for the size of one block or less,
+ * at block-aligned offsets.
+ * Always requests full blocks form the lower layer, truncates the
+ * returned data as neccessary.
+ */
 ssize_t BlockFileIO::cacheReadOneBlock(const IORequest &req) const {
-  // we can satisfy the request even if _cache.dataLen is too short, because
+
-  // we always request a full block during reads..
+  rAssert(req.dataLen <= _blockSize);
-  if ((req.offset == _cache.offset) && (_cache.dataLen != 0)) {
+  rAssert(req.offset % _blockSize == 0);
+
+  /* we can satisfy the request even if _cache.dataLen is too short, because
+   * we always request a full block during reads. This just means we are
+   * in the last block of a file, which may be smaller than the blocksize.
+   * For reverse encryption, the cache must not be used at all, because
+   * the lower file may have changed behind our back. */
+  if ( (_noCache == false) && (req.offset == _cache.offset) &&
+       (_cache.dataLen != 0)) {
     // satisfy request from cache
     int len = req.dataLen;
-    if (_cache.dataLen < len) len = _cache.dataLen;
+    if (_cache.dataLen < len) len = _cache.dataLen; // Don't read past EOF
     memcpy(req.data, _cache.data, len);
     return len;
   } else {
@@ -88,6 +105,13 @@ bool BlockFileIO::cacheWriteOneBlock(const IORequest &req) {
   return ok;
 }
 
+/**
+ * Serve a read request of arbitrary size at an arbitrary offset.
+ * Stitches together multiple blocks to serve large requests, drops
+ * data from the front of the first block if the request is not aligned.
+ * Always requests aligned data of the size of one block or less from the
+ * lower layer.
+ */
 ssize_t BlockFileIO::read(const IORequest &req) const {
   rAssert(_blockSize != 0);
 
@@ -97,7 +121,7 @@ ssize_t BlockFileIO::read(const IORequest &req) const {
 
   if (partialOffset == 0 && req.dataLen <= _blockSize) {
     // read completely within a single block -- can be handled as-is by
-    // readOneBloc().
+    // readOneBlock().
     return cacheReadOneBlock(req);
   } else {
     size_t size = req.dataLen;

encfs/BlockFileIO.h

@@ -57,6 +57,7 @@ class BlockFileIO : public FileIO {
 
   int _blockSize;
   bool _allowHoles;
+  bool _noCache;
 
   // cache last block for speed...
   mutable IORequest _cache;

encfs/BlockNameIO.cpp

@@ -184,7 +184,11 @@ int BlockNameIO::decodeName(const char *encodedName, int length, uint64_t *iv,
   int decodedStreamLen = decLen256 - 2;
 
   // don't bother trying to decode files which are too small
-  if (decodedStreamLen < _bs) throw ERROR("Filename too small to decode");
+  if (decodedStreamLen < _bs)
+  {
+    rDebug("Rejecting filename '%s'", encodedName);
+    throw ERROR("Filename too small to decode");
+  }
 
   BUFFER_INIT(tmpBuf, 32, (unsigned int)length);
 

encfs/CipherFileIO.cpp

@@ -28,6 +28,9 @@
 
 #include <fcntl.h>
 #include <cerrno>
+#include <string.h>
+
+#include <openssl/sha.h>
 
 /*
     - Version 2:0 adds support for a per-file initialization vector with a
@@ -128,26 +131,56 @@ bool CipherFileIO::setIV(uint64_t iv) {
   return base->setIV(iv);
 }
 
+/**
+ * Get file attributes (FUSE-speak for "stat()") for an upper file
+ * Upper file   = file we present to the user via FUSE
+ * Backing file = file that is actually on disk
+ */
 int CipherFileIO::getAttr(struct stat *stbuf) const {
+
+  // stat() the backing file
   int res = base->getAttr(stbuf);
+
   // adjust size if we have a file header
   if ((res == 0) && haveHeader && S_ISREG(stbuf->st_mode) &&
       (stbuf->st_size > 0)) {
+    if(!fsConfig->reverseEncryption)
+    {
+      /* In normal mode, the upper file (plaintext) is smaller
+       * than the backing ciphertext file */
       rAssert(stbuf->st_size >= HEADER_SIZE);
       stbuf->st_size -= HEADER_SIZE;
     }
+    else
+    {
+      /* In reverse mode, the upper file (ciphertext) is larger than
+       * the backing plaintext file */
+      stbuf->st_size += HEADER_SIZE;
+    }
+  }
 
   return res;
 }
 
+/**
+ * Get the size for an upper file
+ * See getAttr() for an explaination of the reverse handling
+ */
 off_t CipherFileIO::getSize() const {
   off_t size = base->getSize();
   // No check on S_ISREG here -- don't call getSize over getAttr unless this
   // is a normal file!
   if (haveHeader && size > 0) {
+    if(!fsConfig->reverseEncryption)
+    {
       rAssert(size >= HEADER_SIZE);
       size -= HEADER_SIZE;
     }
+    else
+    {
+      size += HEADER_SIZE;
+    }
+  }
   return size;
 }
 
@@ -233,15 +266,71 @@ bool CipherFileIO::writeHeader() {
   return true;
 }
 
+/**
+ * Generate the file IV header bytes for reverse mode
+ * (truncated SHA1 hash of the inode number)
+ *
+ * The kernel guarantees that the inode number is unique for one file
+ * system. SHA1 spreads out the values over the whole 64-bit space.
+ * Without this step, the XOR with the block number (see readOneBlock)
+ * may lead to duplicate IVs.
+ * SSL_Cipher::setIVec does an additional HMAC before using
+ * the IV. This guarantees unpredictability and prevents watermarking
+ * attacks.
+ */
+void CipherFileIO::generateReverseHeader(unsigned char* headerBuf) {
+
+  struct stat stbuf;
+  int res = getAttr(&stbuf);
+  rAssert( res == 0 );
+  ino_t ino = stbuf.st_ino;
+  rAssert( ino != 0 );
+
+  rDebug("generating reverse file IV header from ino=%lu", (unsigned long)ino);
+
+  // Serialize the inode number into inoBuf
+  unsigned char inoBuf[sizeof(ino_t)];
+  for (unsigned int i = 0; i < sizeof(ino_t); ++i) {
+    inoBuf[i] = (unsigned char)(ino & 0xff);
+    ino >>= 8;
+  }
+
+  /* Take the SHA1 hash of the inode number so the values are spread out
+   * over the whole 64-bit space. Otherwise, the XOR with the block number
+   * may lead to duplicate IVs (see readOneBlock) */
+  unsigned char md[20];
+  SHA1(inoBuf, sizeof(ino), md);
+  rAssert( HEADER_SIZE <= 20 );
+  memcpy(headerBuf, md, HEADER_SIZE);
+
+  // Save the IV in fileIV for internal use
+  fileIV = 0;
+  for (int i = 0; i < HEADER_SIZE; ++i) {
+    fileIV = (fileIV << 8) | (uint64_t)headerBuf[i];
+  }
+
+  rDebug("fileIV=%" PRIx64, fileIV);
+
+  // Encrypt externally-visible header
+  cipher->streamEncode(headerBuf, HEADER_SIZE, externalIV, key);
+}
+
+/**
+ * Read block from backing ciphertext file, decrypt it (normal mode)
+ * or
+ * Read block from backing plaintext file, then encrypt it (reverse mode)
+ */
 ssize_t CipherFileIO::readOneBlock(const IORequest &req) const {
-  // read raw data, then decipher it..
   int bs = blockSize();
   off_t blockNum = req.offset / bs;
 
   ssize_t readSize = 0;
   IORequest tmpReq = req;
 
-  if (haveHeader) tmpReq.offset += HEADER_SIZE;
+  // adjust offset if we have a file header
+  if (haveHeader && !fsConfig->reverseEncryption) {
+      tmpReq.offset += HEADER_SIZE;
+  }
   readSize = base->read(tmpReq);
 
   bool ok;
@@ -250,6 +339,7 @@ ssize_t CipherFileIO::readOneBlock(const IORequest &req) const {
       const_cast<CipherFileIO *>(this)->initHeader();
 
     if (readSize != bs) {
+      rDebug("streamRead(data, %d, IV)", (int)readSize);
       ok = streamRead(tmpReq.data, (int)readSize, blockNum ^ fileIV);
     } else {
       ok = blockRead(tmpReq.data, (int)readSize, blockNum ^ fileIV);
@@ -267,6 +357,12 @@ ssize_t CipherFileIO::readOneBlock(const IORequest &req) const {
 }
 
 bool CipherFileIO::writeOneBlock(const IORequest &req) {
+
+  if (haveHeader && fsConfig->reverseEncryption) {
+    rDebug("writing to a reverse mount with per-file IVs is not implemented");
+    return -EROFS;
+  }
+
   int bs = blockSize();
   off_t blockNum = req.offset / bs;
 
@@ -296,6 +392,7 @@ bool CipherFileIO::writeOneBlock(const IORequest &req) {
 
 bool CipherFileIO::blockWrite(unsigned char *buf, int size,
                               uint64_t _iv64) const {
+  rDebug("Called blockWrite");
   if (!fsConfig->reverseEncryption)
     return cipher->blockEncode(buf, size, _iv64, key);
   else
@@ -304,6 +401,7 @@ bool CipherFileIO::blockWrite(unsigned char *buf, int size,
 
 bool CipherFileIO::streamWrite(unsigned char *buf, int size,
                                uint64_t _iv64) const {
+  rDebug("Called streamWrite");
   if (!fsConfig->reverseEncryption)
     return cipher->streamEncode(buf, size, _iv64, key);
   else
@@ -359,4 +457,70 @@ int CipherFileIO::truncate(off_t size) {
   return res;
 }
 
+/**
+ * Handle reads for reverse mode with uniqueIV
+ */
+ssize_t CipherFileIO::read(const IORequest &origReq) const {
+
+  /* if reverse mode is not active with uniqueIV,
+   * the read request is handled by the base class */
+  if ( !(fsConfig->reverseEncryption && haveHeader) ) {
+    rDebug("relaying request to base class: offset=%d, dataLen=%d", origReq.offset, origReq.dataLen);
+    return BlockFileIO::read(origReq);
+  }
+
+  rDebug("handling reverse unique IV read: offset=%d, dataLen=%d", origReq.offset, origReq.dataLen);
+
+  // generate the file IV header
+  // this is needed in any case - without IV the file cannot be decoded
+  unsigned char headerBuf[HEADER_SIZE];
+  const_cast<CipherFileIO *>(this)->generateReverseHeader(headerBuf);
+
+  // Copy the request so we can modify it without affecting the caller
+  IORequest req = origReq;
+
+  /* An offset x in the ciphertext file maps to x-8 in the
+   * plain text file. Values below zero are the header. */
+  req.offset -= HEADER_SIZE;
+
+  int headerBytes = 0; // number of header bytes to add
+
+  /* The request contains (a part of) the header, so we prefix that part
+   * to the data. */
+  if (req.offset < 0) {
+    headerBytes = -req.offset;
+    if ( req.dataLen < headerBytes )
+      headerBytes = req.dataLen; // only up to the number of bytes requested
+    rDebug("Adding %d header bytes", headerBytes);
+
+    // copy the header bytes into the data
+    int headerOffset = HEADER_SIZE - headerBytes;
+    memcpy(req.data, &headerBuf[headerOffset], headerBytes);
+
+    // the read does not want data beyond the header
+    if ( headerBytes == req.dataLen)
+      return headerBytes;
+
+    /* The rest of the request will be read from the backing file.
+     * As we have already generated n=headerBytes bytes, the request is
+     * shifted by headerBytes */
+    req.offset += headerBytes;
+    rAssert( req.offset == 0 );
+    req.data += headerBytes;
+    req.dataLen -= headerBytes;
+  }
+
+  // read the payload
+  ssize_t readBytes = BlockFileIO::read(req);
+  rDebug("read %ld bytes from backing file", (long)readBytes);
+  if ( readBytes < 0)
+    return readBytes; // Return error code
+  else
+  {
+    ssize_t sum = headerBytes + readBytes;
+    rDebug("returning sum=%ld", (long)sum);
+    return sum;
+  }
+}
+
 bool CipherFileIO::isWritable() const { return base->isWritable(); }

encfs/CipherFileIO.h

@@ -57,6 +57,7 @@ class CipherFileIO : public BlockFileIO {
  private:
   virtual ssize_t readOneBlock(const IORequest &req) const;
   virtual bool writeOneBlock(const IORequest &req);
+  virtual void generateReverseHeader(unsigned char* data);
 
   void initHeader();
   bool writeHeader();
@@ -65,6 +66,8 @@ class CipherFileIO : public BlockFileIO {
   bool blockWrite(unsigned char *buf, int size, uint64_t iv64) const;
   bool streamWrite(unsigned char *buf, int size, uint64_t iv64) const;
 
+  ssize_t read(const IORequest &req) const;
+
   shared_ptr<FileIO> base;
 
   FSConfigPtr fsConfig;

encfs/FileUtils.cpp

@@ -99,6 +99,11 @@ static int V5SubVersionDefault = 0;
 // 20080813 was really made on 20080413 -- typo on date..
 // const int V6SubVersion = 20080813; // switch to v6/XML, add allowHoles option
 // const int V6SubVersion = 20080816; // add salt and iteration count
+/*
+ * In boost 1.42+, serial numbers change to 8 bit, which means the date
+ * numbering scheme does not work any longer.
+ * boost-versioning.h implements a workaround that sets the version to
+ * 20 for boost 1.42+. */
 const int V6SubVersion = 20100713;  // add version field for boost 1.42+
 
 struct ConfigInfo {
@@ -111,6 +116,7 @@ struct ConfigInfo {
   int currentSubVersion;
   int defaultSubVersion;
 } ConfigFileMapping[] = {
+      // current format
       {".encfs6.xml", Config_V6, "ENCFS6_CONFIG", readV6Config, writeV6Config,
        V6SubVersion, 0},
       // backward compatible support for older versions
@@ -321,6 +327,9 @@ bool userAllowMkdir(int promptno, const char *path, mode_t mode) {
   }
 }
 
+/**
+ * Load config file by calling the load function on the filename
+ */
 ConfigType readConfig_load(ConfigInfo *nm, const char *path,
                            const shared_ptr<EncFSConfig> &config) {
   if (nm->loadFunc) {
@@ -334,8 +343,8 @@ ConfigType readConfig_load(ConfigInfo *nm, const char *path,
       err.log(_RLWarningChannel);
     }
 
-    rError(_("Found config file %s, but failed to load"), path);
+    rError(_("Found config file %s, but failed to load - exiting"), path);
-    return Config_None;
+    exit(1);
   } else {
     // No load function - must be an unsupported type..
     config->cfgType = nm->type;
@@ -343,6 +352,10 @@ ConfigType readConfig_load(ConfigInfo *nm, const char *path,
   }
 }
 
+/**
+ * Try to locate the config file
+ * Tries the most recent format first, then looks for older versions
+ */
 ConfigType readConfig(const string &rootDir,
                       const shared_ptr<EncFSConfig> &config) {
   ConfigInfo *nm = ConfigFileMapping;
@@ -350,7 +363,13 @@ ConfigType readConfig(const string &rootDir,
     // allow environment variable to override default config path
     if (nm->environmentOverride != NULL) {
       char *envFile = getenv(nm->environmentOverride);
-      if (envFile != NULL) return readConfig_load(nm, envFile, config);
+      if (envFile != NULL) {
+        if (! fileExists(envFile)) {
+          rError("fatal: config file specified by environment does not exist: %s", envFile);
+          exit(1);
+        }
+        return readConfig_load(nm, envFile, config);
+      }
     }
     // the standard place to look is in the root directory
     string path = rootDir + nm->fileName;
@@ -363,6 +382,10 @@ ConfigType readConfig(const string &rootDir,
   return Config_None;
 }
 
+/**
+ * Read config file in current "V6" XML format, normally named ".encfs6.xml"
+ * This format is in use since Apr 13, 2008 (commit 6d081f5c)
+ */
 bool readV6Config(const char *configFile, const shared_ptr<EncFSConfig> &config,
                   ConfigInfo *info) {
   (void)info;
@@ -385,6 +408,10 @@ bool readV6Config(const char *configFile, const shared_ptr<EncFSConfig> &config,
   }
 }
 
+/**
+ * Read config file in deprecated "V5" format, normally named ".encfs5"
+ * This format has been used before Apr 13, 2008
+ */
 bool readV5Config(const char *configFile, const shared_ptr<EncFSConfig> &config,
                   ConfigInfo *info) {
   bool ok = false;
@@ -437,6 +464,10 @@ bool readV5Config(const char *configFile, const shared_ptr<EncFSConfig> &config,
   return ok;
 }
 
+/**
+ * Read config file in deprecated "V4" format, normally named ".encfs4"
+ * This format has been used before Jan 7, 2008
+ */
 bool readV4Config(const char *configFile, const shared_ptr<EncFSConfig> &config,
                   ConfigInfo *info) {
   bool ok = false;
@@ -576,6 +607,9 @@ static Cipher::CipherAlgorithm findCipherAlgorithm(const char *name,
   return result;
 }
 
+/**
+ * Ask the user which cipher to use
+ */
 static Cipher::CipherAlgorithm selectCipherAlgorithm() {
   for (;;) {
     // figure out what cipher they want to use..
@@ -639,6 +673,9 @@ static Cipher::CipherAlgorithm selectCipherAlgorithm() {
   }
 }
 
+/**
+ * Ask the user which encoding to use for file names
+ */
 static Interface selectNameCoding() {
   for (;;) {
     // figure out what cipher they want to use..
@@ -676,6 +713,9 @@ static Interface selectNameCoding() {
   }
 }
 
+/**
+ * Ask the user which key size to use
+ */
 static int selectKeySize(const Cipher::CipherAlgorithm &alg) {
   if (alg.keyLength.min() == alg.keyLength.max()) {
     cout << autosprintf(_("Using key size of %i bits"), alg.keyLength.min())
@@ -726,6 +766,9 @@ static int selectKeySize(const Cipher::CipherAlgorithm &alg) {
   return keySize;
 }
 
+/**
+ * Ask the user which block size to use
+ */
 static int selectBlockSize(const Cipher::CipherAlgorithm &alg) {
   if (alg.blockSize.min() == alg.blockSize.max()) {
     cout << autosprintf(
@@ -777,6 +820,9 @@ static bool boolDefaultNo(const char *prompt) {
     return false;
 }
 
+/**
+ * Ask the user whether to enable block MAC and random header bytes
+ */
 static void selectBlockMAC(int *macBytes, int *macRandBytes) {
   // xgroup(setup)
   bool addMAC = boolDefaultNo(
@@ -827,6 +873,9 @@ static bool boolDefaultYes(const char *prompt) {
     return true;
 }
 
+/**
+ * Ask the user if per-file unique IVs should be used
+ */
 static bool selectUniqueIV() {
   // xgroup(setup)
   return boolDefaultYes(
@@ -836,6 +885,9 @@ static bool selectUniqueIV() {
         "which rely on block-aligned file io for performance."));
 }
 
+/**
+ * Ask the user if the filename IV should depend on the complete path
+ */
 static bool selectChainedIV() {
   // xgroup(setup)
   return boolDefaultYes(
@@ -844,6 +896,9 @@ static bool selectChainedIV() {
         "rather then encoding each path element individually."));
 }
 
+/**
+ * Ask the user if the file IV should depend on the file path
+ */
 static bool selectExternalChainedIV() {
   // xgroup(setup)
   return boolDefaultNo(
@@ -855,6 +910,9 @@ static bool selectExternalChainedIV() {
         "in the filesystem."));
 }
 
+/**
+ * Ask the user if file holes should be passed through
+ */
 static bool selectZeroBlockPassThrough() {
   // xgroup(setup)
   return boolDefaultYes(
@@ -895,16 +953,17 @@ RootPtr createV6Config(EncFS_Context *ctx, const shared_ptr<EncFS_Opts> &opts) {
     cout << "\n";
   }
 
-  int keySize = 0;
+  //                          documented in ...
-  int blockSize = 0;
+  int keySize = 0; //                       selectKeySize()
-  Cipher::CipherAlgorithm alg;
+  int blockSize = 0; //                     selectBlockSize()
-  Interface nameIOIface;
+  Cipher::CipherAlgorithm alg; //           selectCipherAlgorithm()
-  int blockMACBytes = 0;
+  Interface nameIOIface; //                 selectNameCoding()
-  int blockMACRandBytes = 0;
+  int blockMACBytes = 0;   //               selectBlockMAC()
-  bool uniqueIV = false;
+  int blockMACRandBytes = 0; //             selectBlockMAC()
-  bool chainedIV = false;
+  bool uniqueIV = false;   //               selectUniqueIV()
-  bool externalIV = false;
+  bool chainedIV = false;  //               selectChainedIV()
-  bool allowHoles = true;
+  bool externalIV = false; //               selectExternalChainedIV()
+  bool allowHoles = true;  //               selectZeroBlockPassThrough()
   long desiredKDFDuration = NormalKDFDuration;
 
   if (reverseEncryption) {
@@ -949,11 +1008,11 @@ RootPtr createV6Config(EncFS_Context *ctx, const shared_ptr<EncFS_Opts> &opts) {
     blockMACBytes = 0;
     externalIV = false;
     nameIOIface = BlockNameIO::CurrentInterface();
+    uniqueIV = true;
 
     if (reverseEncryption) {
-      cout << _("--reverse specified, not using unique/chained IV") << "\n";
+      cout << _("--reverse specified, not using chained IV") << "\n";
     } else {
-      uniqueIV = true;
       chainedIV = true;
     }
   }
@@ -1421,7 +1480,7 @@ RootPtr initFS(EncFS_Context *ctx, const shared_ptr<EncFS_Opts> &opts) {
   if (readConfig(opts->rootDir, config) != Config_None) {
     if (opts->reverseEncryption) {
       if (config->blockMACBytes != 0 || config->blockMACRandBytes != 0 ||
-          config->uniqueIV || config->externalIVChaining ||
+          config->externalIVChaining ||
           config->chainedNameIV) {
         cout
             << _("The configuration loaded is not compatible with --reverse\n");

encfs/FileUtils.h

@@ -76,6 +76,11 @@ struct EncFS_Opts {
 
   bool reverseEncryption;  // Reverse encryption
 
+  bool noCache; /* Disable block cache (in EncFS) and stat cache (in kernel).
+                 * This is needed if the backing files may be modified
+                 * behind the back of EncFS (for example, in reverse mode).
+                 * See main.cpp for a longer explaination. */
+
   ConfigMode configMode;
 
   EncFS_Opts() {
@@ -90,6 +95,7 @@ struct EncFS_Opts {
     ownerCreate = false;
     reverseEncryption = false;
     configMode = Config_Prompt;
+    noCache = false;
   }
 };
 

encfs/SSL_Cipher.cpp

@@ -49,14 +49,14 @@ using namespace rel;
 using namespace rlog;
 
 const int MAX_KEYLENGTH = 32;  // in bytes (256 bit)
-const int MAX_IVLENGTH = 16;
+const int MAX_IVLENGTH = 16; // 128 bit (AES block size, Blowfish has 64)
 const int KEY_CHECKSUM_BYTES = 4;
 
 #ifndef MIN
 inline int MIN(int a, int b) { return (a < b) ? a : b; }
 #endif
 
-/*
+/**
     This produces the same result as OpenSSL's EVP_BytesToKey.  The difference
     is that here we can explicitly specify the key size, instead of relying on
     the state of EVP_CIPHER struct.  EVP_BytesToKey will only produce 128 bit
@@ -348,7 +348,7 @@ SSL_Cipher::~SSL_Cipher() {}
 
 Interface SSL_Cipher::interface() const { return realIface; }
 
-/*
+/**
     create a key from the password.
     Use SHA to distribute entropy from the password into the key.
 
@@ -413,7 +413,7 @@ CipherKey SSL_Cipher::newKey(const char *password, int passwdLength) {
   return key;
 }
 
-/*
+/**
     Create a random key.
     We use the OpenSSL library to generate random bytes, then take the hash of
     those bytes to use as the key.
@@ -447,7 +447,7 @@ CipherKey SSL_Cipher::newRandomKey() {
   return key;
 }
 
-/*
+/**
     compute a 64-bit check value for the data using HMAC.
 */
 static uint64_t _checksum_64(SSLKey *key, const unsigned char *data,
@@ -487,6 +487,11 @@ static uint64_t _checksum_64(SSLKey *key, const unsigned char *data,
   return value;
 }
 
+/**
+ * Write "len" bytes of random data into "buf"
+ *
+ * See "man 3 RAND_bytes" for the effect of strongRandom
+ */
 bool SSL_Cipher::randomize(unsigned char *buf, int len,
                            bool strongRandom) const {
   // to avoid warnings of uninitialized data from valgrind
@@ -604,6 +609,22 @@ int SSL_Cipher::cipherBlockSize() const {
   return EVP_CIPHER_block_size(_blockCipher);
 }
 
+/**
+ * Generate the initialization vector that will actually be used for
+ * AES/Blowfish encryption and decryption in {stream,block}{Encode,Decode}
+ *
+ * It is derived from
+ *  1) a "seed" value that is passed from the higher layer, for the default
+ *     configuration it is "block_number XOR per_file_IV_header" from
+ *     CipherFileIO
+ *  2) The IV that is used for encrypting the master key, "IVData(key)"
+ *  3) The master key
+ * using
+ *  ivec = HMAC(master_key, IVData(key) CONCAT seed)
+ *
+ * As an HMAC is unpredictable as long as the key is secret, the only
+ * requirement for "seed" is that is must be unique.
+ */
 void SSL_Cipher::setIVec(unsigned char *ivec, uint64_t seed,
                          const shared_ptr<SSLKey> &key) const {
   if (iface.current() >= 3) {
@@ -695,7 +716,7 @@ static void unshuffleBytes(unsigned char *buf, int size) {
   for (int i = size - 1; i; --i) buf[i] ^= buf[i - 1];
 }
 
-/* Partial blocks are encoded with a stream cipher.  We make multiple passes on
+/** Partial blocks are encoded with a stream cipher.  We make multiple passes on
  the data to ensure that the ends of the data depend on each other.
 */
 bool SSL_Cipher::streamEncode(unsigned char *buf, int size, uint64_t iv64,

encfs/encfs.cpp

@@ -93,7 +93,7 @@ static int withCipherPath(const char *opName, const char *path,
     } else if (!passReturnCode)
       res = ESUCCESS;
   } catch (rlog::Error &err) {
-    rError("error caught in %s", opName);
+    rError("withCipherPath: error caught in %s: %s", opName, err.message());
     err.log(_RLWarningChannel);
   }
   return res;
@@ -123,7 +123,7 @@ static int withFileNode(const char *opName, const char *path,
 
     if (res < 0) rInfo("%s error: %s", opName, strerror(-res));
   } catch (rlog::Error &err) {
-    rError("error caught in %s", opName);
+    rError("withFileNode: error caught in %s: %s", opName, err.message());
     err.log(_RLWarningChannel);
   }
   return res;

encfs/main.cpp

@@ -212,6 +212,7 @@ static bool processArgs(int argc, char *argv[],
       // {"single-thread", 0, 0, 's'}, // single-threaded mode
       {"stdinpass", 0, 0, 'S'},  // read password from stdin
       {"annotate", 0, 0, 513},   // Print annotation lines to stderr
+      {"nocache", 0, 0, 514},    // disable caching
       {"verbose", 0, 0, 'v'},    // verbose mode
       {"version", 0, 0, 'V'},    // version
       {"reverse", 0, 0, 'r'},    // reverse encryption
@@ -272,8 +273,32 @@ static bool processArgs(int argc, char *argv[],
       case 'D':
         out->opts->forceDecode = true;
         break;
+      /* By default, the kernel caches file metadata for one second.
+       * This is fine for EncFS' normal mode, but for --reverse, this
+       * means that the encrypted view will be up to one second out of
+       * date.
+       * Quoting Goswin von Brederlow:
+       * "Caching only works correctly if you implement a disk based
+       * filesystem, one where only the fuse process can alter
+       * metadata and all access goes only through fuse. Any overlay
+       * filesystem where something can change the underlying
+       * filesystem without going through fuse can run into
+       * inconsistencies."
+       * Enabling reverse automatically enables noCache. */
       case 'r':
         out->opts->reverseEncryption = true;
+      case 514:
+        /* Disable EncFS block cache
+         * Causes reverse grow tests to fail because short reads
+         * are returned */
+        out->opts->noCache = true;
+        /* Disable kernel stat() cache
+         * Causes reverse grow tests to fail because stale stat() data
+         * is returned */
+        PUSHARG("-oattr_timeout=0");
+        /* Disable kernel dentry cache
+         * Fallout unknown, disabling for safety */
+        PUSHARG("-oentry_timeout=0");
         break;
       case 'm':
         out->opts->mountOnDemand = true;

tests/common.inc

@@ -0,0 +1,76 @@
+# Helper function
+# Get the MD5 sum of the file open at the filehandle
+use Digest::MD5 qw(md5_hex);
+sub md5fh
+{
+    my $fh_orig = shift;
+    open(my $fh, "<&", $fh_orig); # Duplicate the file handle so the seek
+    seek($fh, 0, 0);              # does not affect the caller
+    my $md5 = Digest::MD5->new->addfile($fh)->hexdigest;
+    close($fh);
+    return $md5;
+}
+
+# Get the file size from stat() (by file handle or name)
+sub statSize
+{
+    my $f = shift;
+    my @s = stat($f) or die("stat on '$f' failed");
+    return $s[7];
+}
+
+# Get the file size by read()ing the whole file
+sub readSize
+{
+   my $fh = shift;
+   seek($fh, 0, 0);
+   my $block = 4*1024;
+   my $s;
+   my $data;
+   my $sum = read($fh, $data, $block);
+   while ( $s = read($fh, $data, $block) )
+   {
+        $sum += $s;
+   }
+   $data = "";
+   return $sum;
+}
+
+# Verify that the size of the file passed by filehandle matches the target size s0
+# Checks both stat() and read()
+sub sizeVerify
+{
+	my $ok = 1;
+	my $fh = shift;
+	my $s0 = shift;
+	$ss = statSize($fh);
+	if ($s0 != $ss) {
+		$ok = 0;
+		print("# stat size $ss, expected $s0\n");
+	}
+	$sr = readSize($fh);
+	if ($s0 != $sr) {
+		$ok = 0;
+		print("# read size $sr, expected $s0\n");
+	}
+	return $ok;
+}
+
+# Wait for a file to appear
+use Time::HiRes qw(usleep);
+sub waitForFile
+{
+	my $file = shift;
+	my $timeout;
+	$timeout = shift or $timeout = 5;
+	for(my $i = $timeout*10; $i > 0; $i--)
+	{
+		-f $file and return 1;
+		usleep(100000); # 0.1 seconds
+	}
+	print "# timeout waiting for '$file' to appear\n";
+	return 0;
+}
+
+# As this file will be require()'d, it needs to return true
+return 1;

tests/normal.pl

@@ -7,7 +7,8 @@ use File::Path;
 use File::Copy;
 use File::Temp;
 use IO::Handle;
-use Digest::MD5 qw(md5_hex);
+
+require("tests/common.inc");
 
 my $tempDir = $ENV{'TMPDIR'} || "/tmp";
 
@@ -190,7 +191,7 @@ sub truncate
 sub fileCreation
 {
     # create a file
-    qx(df -ah > "$crypt/df.txt");
+    qx(df -ah > "$crypt/df.txt" 2> /dev/null);
     ok( -f "$crypt/df.txt", "file created" );
 
     # ensure there is an encrypted version.
@@ -268,17 +269,6 @@ sub encName
     return $enc;
 }
 
-# Helper function
-# Get the MD5 sum of the file open at the filehandle
-sub md5fh
-{
-    my $fh_orig = shift;
-    open(my $fh, "<&", $fh_orig); # Duplicate the file handle so the seek
-    seek($fh, 0, 0);              # does not affect the caller
-    return Digest::MD5->new->addfile($fh)->hexdigest;
-    close($fh);
-}
-
 # Test symlinks & hardlinks
 sub links
 {

tests/reverse.pl

@@ -2,9 +2,13 @@
 
 # Test EncFS --reverse mode
 
+use warnings;
 use Test::More qw( no_plan );
 use File::Path;
 use File::Temp;
+use IO::Handle;
+
+require("tests/common.inc");
 
 my $tempDir = $ENV{'TMPDIR'} || "/tmp";
 
@@ -29,7 +33,7 @@ sub cleanup
 {
     system("fusermount -u $decrypted");
     system("fusermount -u $ciphertext");
-
+    our $workingDir;
     rmtree($workingDir);
     ok( ! -d $workingDir, "working dir removed");
 }
@@ -40,11 +44,23 @@ sub cleanup
 # Directory structure: plain -[encrypt]-> ciphertext -[decrypt]-> decrypted
 sub mount
 {
-    my $r=system("./encfs/encfs --extpass=\"echo test\" --standard $plain $ciphertext --reverse > /dev/null");
+    system("./encfs/encfs --extpass=\"echo test\" --standard $plain $ciphertext --reverse");
-    ok($r == 0, "mounted ciphertext file system");
+    ok(waitForFile("$plain/.encfs6.xml"), "plain .encfs6.xml exists") or BAIL_OUT("'$plain/.encfs6.xml'");
+    my $e = encName(".encfs6.xml");
+    ok(waitForFile("$ciphertext/$e"), "encrypted .encfs6.xml exists") or BAIL_OUT("'$ciphertext/$e'");
+    system("ENCFS6_CONFIG=$plain/.encfs6.xml ./encfs/encfs --nocache --extpass=\"echo test\" $ciphertext $decrypted");
+    ok(waitForFile("$decrypted/.encfs6.xml"), "decrypted .encfs6.xml exists") or BAIL_OUT("'$decrypted/.encfs6.xml'");
+}
 
-    $r=system("ENCFS6_CONFIG=$plain/.encfs6.xml ./encfs/encfs --extpass=\"echo test\" $ciphertext $decrypted");
+# Helper function
-    ok($r == 0, "mounted decrypting file system");
+#
+# Get encrypted name for file
+sub encName
+{
+	my $name = shift;
+	my $enc = qx(ENCFS6_CONFIG=$plain/.encfs6.xml ./encfs/encfsctl encode --extpass="echo test" $ciphertext $name);
+	chomp($enc);
+	return $enc;
 }
 
 # Copy a directory tree and verify that the decrypted data is identical
@@ -58,9 +74,81 @@ sub copy_test
     ok(! -f "$decrypted/encfs.cpp", "file deleted");
 }
 
+# Create symlinks and verify they are correctly decrypted
+# Parameter: symlink target
+sub symlink_test
+{
+    my $target = shift(@_);
+    symlink($target, "$plain/symlink");
+    ok( readlink("$decrypted/symlink") eq "$target", "symlink to '$target'");
+    unlink("$plain/symlink");
+}
+
+# Grow a file from 0 to x kB and
+# * check the ciphertext length is correct (stat + read)
+# * check that the decrypted length is correct (stat + read)
+# * check that plaintext and decrypted are identical
+sub grow {
+    # pfh ... plaintext file handle
+    open(my $pfh, ">", "$plain/grow");
+    # vfh ... verification file handle
+    open(my $vfh, "<", "$plain/grow");
+    $pfh->autoflush;
+    # ciphertext file name
+    my $cname = encName("grow");
+    # cfh ... ciphertext file handle
+    ok(open(my $cfh, "<", "$ciphertext/$cname"), "open ciphertext grow file");
+    # dfh ... decrypted file handle
+    ok(open(my $dfh, "<", "$decrypted/grow"), "open decrypted grow file");
+
+    # csz ... ciphertext size
+    ok(sizeVerify($cfh, 0), "ciphertext of empty file is empty");
+    ok(sizeVerify($dfh, 0), "decrypted empty file is empty");
+
+    my $ok = 1;
+    my $max = 9000;
+    for($i=5; $i < $max; $i += 5)
+    {
+        print($pfh "abcde") or die("write failed");
+        # autoflush should make sure the write goes to the kernel
+        # immediately. Just to be sure, check it here.
+        sizeVerify($vfh, $i) or die("unexpected plain file size");
+        sizeVerify($cfh, $i+8) or $ok = 0;
+        sizeVerify($dfh, $i) or $ok = 0;
+        
+        if(md5fh($vfh) ne md5fh($dfh))
+        {
+            $ok = 0;
+            print("# content is different, unified diff:\n");
+            system("diff -u $plain/grow $decrypted/grow");
+        }
+
+        last unless $ok;
+    }
+    ok($ok, "ciphertext and decrypted size of file grown to $i bytes");
+}
+
+sub largeRead {
+    system("dd if=/dev/zero of=$plain/largeRead bs=1M count=1 2> /dev/null");
+    # ciphertext file name
+    my $cname = encName("largeRead");
+    # cfh ... ciphertext file handle
+    ok(open(my $cfh, "<", "$ciphertext/$cname"), "open ciphertext largeRead file");
+    ok(sizeVerify($cfh, 1024*1024+8), "1M file size");
+}
+
+# Setup mounts
 newWorkingDir();
 mount();
 
+# Actual tests
+grow();
+largeRead();
 copy_test();
+symlink_test("/"); # absolute
+symlink_test("foo"); # relative
+symlink_test("/1/2/3/4/5/6/7/8/9/10/11/12/13/14/15/15/17/18"); # long
+symlink_test("!§\$%&/()\\<>#+="); # special characters
 
+# Umount and delete files
 cleanup();