src/test/java/com/google/devtools/build/lib/actions/DigestUtilsTest.java - bazel - Git at Google

 // Copyright 2015 The Bazel Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 package com.google.devtools.build.lib.actions;


 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;

 import com.google.common.base.Strings;
 import com.google.common.cache.CacheStats;
 import com.google.devtools.build.lib.actions.cache.DigestUtils;
 import com.google.devtools.build.lib.testutil.TestThread;
 import com.google.devtools.build.lib.testutil.TestUtils;
 import com.google.devtools.build.lib.util.BlazeClock;
 import com.google.devtools.build.lib.vfs.FileSystem;
 import com.google.devtools.build.lib.vfs.FileSystem.HashFunction;
 import com.google.devtools.build.lib.vfs.FileSystemUtils;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import javax.annotation.CheckReturnValue;
 import org.junit.After;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;

 /**
  * Tests for DigestUtils.
  */
 @RunWith(JUnit4.class)
 public class DigestUtilsTest {

   @After
   public void tearDown() {
     DigestUtils.configureCache(0);
   }

   private static void assertDigestCalculationConcurrency(boolean expectConcurrent,
       final boolean fastDigest, final int fileSize1, final int fileSize2,
       HashFunction hf) throws Exception {
     final CountDownLatch barrierLatch = new CountDownLatch(2); // Used to block test threads.
     final CountDownLatch readyLatch = new CountDownLatch(1);   // Used to block main thread.

     FileSystem myfs = new InMemoryFileSystem(BlazeClock.instance()) {
         @Override
         protected byte[] getMD5Digest(Path path) throws IOException {
           try {
             barrierLatch.countDown();
             readyLatch.countDown();
             // Either both threads will be inside getMD5Digest at the same time or they
             // both will be blocked.
             barrierLatch.await();
           } catch (Exception e) {
             throw new IOException(e);
           }
           return super.getMD5Digest(path);
         }

         @Override
         protected byte[] getSHA1Digest(Path path) throws IOException {
           try {
             barrierLatch.countDown();
             readyLatch.countDown();
             // Either both threads will be inside getSHA1Digest at the same time or they
             // both will be blocked.
             barrierLatch.await();
           } catch (Exception e) {
             throw new IOException(e);
           }
           return super.getSHA1Digest(path);
         }

         @Override
         protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException {
           return fastDigest ? super.getDigest(path, hashFunction) : null;
         }
     };

     FileSystem.setDigestFunctionForTesting(hf);
     final Path myFile1 = myfs.getPath("/f1.dat");
     final Path myFile2 = myfs.getPath("/f2.dat");
     FileSystemUtils.writeContentAsLatin1(myFile1, Strings.repeat("a", fileSize1));
     FileSystemUtils.writeContentAsLatin1(myFile2, Strings.repeat("b", fileSize2));

      TestThread thread1 = new TestThread () {
        @Override public void runTest() throws Exception {
          DigestUtils.getDigestOrFail(myFile1, fileSize1);
        }
      };

      TestThread thread2 = new TestThread () {
        @Override public void runTest() throws Exception {
          DigestUtils.getDigestOrFail(myFile2, fileSize2);
        }
      };

      thread1.start();
      thread2.start();
      if (!expectConcurrent) { // Synchronized case.
        // Wait until at least one thread reached getMD5Digest().
        assertTrue(readyLatch.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
        // Only 1 thread should be inside getMD5Digest().
        assertEquals(1, barrierLatch.getCount());
        barrierLatch.countDown(); // Release barrier latch, allowing both threads to proceed.
      }
      // Test successful execution within 5 seconds.
      thread1.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
      thread2.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
   }

   /**
    * Ensures that MD5 calculation is synchronized for files
    * greater than 4096 bytes if MD5 is not available cheaply,
    * so machines with rotating drives don't become unusable.
    */
   @Test
   public void testCalculationConcurrency() throws Exception {
     for (HashFunction hf : Arrays.asList(HashFunction.MD5, HashFunction.SHA1)) {
       assertDigestCalculationConcurrency(true, true, 4096, 4096, hf);
       assertDigestCalculationConcurrency(true, true, 4097, 4097, hf);
       assertDigestCalculationConcurrency(true, false, 4096, 4096, hf);
       assertDigestCalculationConcurrency(false, false, 4097, 4097, hf);
       assertDigestCalculationConcurrency(true, false, 1024, 4097, hf);
       assertDigestCalculationConcurrency(true, false, 1024, 1024, hf);
     }
   }

   public void assertRecoverFromMalformedDigest(HashFunction... hashFunctions) throws Exception {
     final byte[] malformed = {0, 0, 0};
     FileSystem myFS = new InMemoryFileSystem(BlazeClock.instance()) {
       @Override
       protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException {
         // Digest functions have more than 3 bytes, usually at least 16.
         return malformed;
       }
     };
     Path path = myFS.getPath("/file");
     FileSystemUtils.writeContentAsLatin1(path, "a");
     for (HashFunction hf : hashFunctions) {
       FileSystem.setDigestFunctionForTesting(hf);
       byte[] result = DigestUtils.getDigestOrFail(path, 1);
       assertArrayEquals(path.getDigest(), result);
       assertNotSame(malformed, result);
       assertTrue(path.isValidDigest(result));
     }
   }

   @Test
   public void testRecoverFromMalformedDigestWithoutCache() throws Exception {
     try {
       DigestUtils.getCacheStats();
       fail("Digests cache should remain disabled until configureCache is called");
     } catch (NullPointerException expected) {
     }
     assertRecoverFromMalformedDigest(HashFunction.MD5, HashFunction.SHA1);
     try {
       DigestUtils.getCacheStats();
       fail("Digests cache was unexpectedly enabled through the test");
     } catch (NullPointerException expected) {
     }
   }

   @Test
   public void testRecoverFromMalformedDigestWithCache() throws Exception {
     DigestUtils.configureCache(10);
     assertNotNull(DigestUtils.getCacheStats()); // Ensure the cache is enabled.

     // When using the cache, we cannot run our test using different hash functions because the
     // hash function is not part of the cache key. This is intentional: the hash function is
     // essentially final and can only be changed for tests. Therefore, just test the same hash
     // function twice to further exercise the cache code.
     assertRecoverFromMalformedDigest(HashFunction.MD5, HashFunction.MD5);

     assertNotNull(DigestUtils.getCacheStats()); // Ensure the cache remains enabled.
   }

   /** Helper class to assert the cache statistics. */
   private static class CacheStatsChecker {
     /** Cache statistics, grabbed at construction time. */
     private final CacheStats stats;

     private int expectedEvictionCount;
     private int expectedHitCount;
     private int expectedMissCount;

     CacheStatsChecker() {
       this.stats = DigestUtils.getCacheStats();
     }

     @CheckReturnValue
     CacheStatsChecker evictionCount(int count) {
       expectedEvictionCount = count;
       return this;
     }

     @CheckReturnValue
     CacheStatsChecker hitCount(int count) {
       expectedHitCount = count;
       return this;
     }

     @CheckReturnValue
     CacheStatsChecker missCount(int count) {
       expectedMissCount = count;
       return this;
     }

     void check() throws Exception {
       assertEquals(expectedEvictionCount, stats.evictionCount());
       assertEquals(expectedHitCount, stats.hitCount());
       assertEquals(expectedMissCount, stats.missCount());
     }
   }

   @Test
   public void testCache() throws Exception {
     final AtomicInteger getFastDigestCounter = new AtomicInteger(0);
     final AtomicInteger getDigestCounter = new AtomicInteger(0);

     FileSystem tracingFileSystem =
         new InMemoryFileSystem(BlazeClock.instance()) {
           @Override
           protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException {
             getFastDigestCounter.incrementAndGet();
             return null;
           }

           @Override
           protected byte[] getDigest(Path path) throws IOException {
             getDigestCounter.incrementAndGet();
             return super.getDigest(path);
           }
         };

     DigestUtils.configureCache(2);

     final Path file1 = tracingFileSystem.getPath("/1.txt");
     final Path file2 = tracingFileSystem.getPath("/2.txt");
     final Path file3 = tracingFileSystem.getPath("/3.txt");
     FileSystemUtils.writeContentAsLatin1(file1, "some contents");
     FileSystemUtils.writeContentAsLatin1(file2, "some other contents");
     FileSystemUtils.writeContentAsLatin1(file3, "and something else");

     byte[] digest1 = DigestUtils.getDigestOrFail(file1, file1.getFileSize());
     assertEquals(1, getFastDigestCounter.get());
     assertEquals(1, getDigestCounter.get());
     new CacheStatsChecker().evictionCount(0).hitCount(0).missCount(1).check();

     byte[] digest2 = DigestUtils.getDigestOrFail(file1, file1.getFileSize());
     assertEquals(2, getFastDigestCounter.get());
     assertEquals(1, getDigestCounter.get());
     new CacheStatsChecker().evictionCount(0).hitCount(1).missCount(1).check();

     assertArrayEquals(digest1, digest2);

     // Evict the digest for the previous file.
     DigestUtils.getDigestOrFail(file2, file2.getFileSize());
     DigestUtils.getDigestOrFail(file3, file3.getFileSize());
     new CacheStatsChecker().evictionCount(1).hitCount(1).missCount(3).check();

     // And now try to recompute it.
     byte[] digest3 = DigestUtils.getDigestOrFail(file1, file1.getFileSize());
     new CacheStatsChecker().evictionCount(2).hitCount(1).missCount(4).check();

     assertArrayEquals(digest1, digest3);
   }
 }
	// Copyright 2015 The Bazel Authors. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.
	package com.google.devtools.build.lib.actions;


	import static org.junit.Assert.assertArrayEquals;
	import static org.junit.Assert.assertEquals;
	import static org.junit.Assert.assertNotNull;
	import static org.junit.Assert.assertNotSame;
	import static org.junit.Assert.assertTrue;
	import static org.junit.Assert.fail;

	import com.google.common.base.Strings;
	import com.google.common.cache.CacheStats;
	import com.google.devtools.build.lib.actions.cache.DigestUtils;
	import com.google.devtools.build.lib.testutil.TestThread;
	import com.google.devtools.build.lib.testutil.TestUtils;
	import com.google.devtools.build.lib.util.BlazeClock;
	import com.google.devtools.build.lib.vfs.FileSystem;
	import com.google.devtools.build.lib.vfs.FileSystem.HashFunction;
	import com.google.devtools.build.lib.vfs.FileSystemUtils;
	import com.google.devtools.build.lib.vfs.Path;
	import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
	import java.io.IOException;
	import java.util.Arrays;
	import java.util.concurrent.CountDownLatch;
	import java.util.concurrent.TimeUnit;
	import java.util.concurrent.atomic.AtomicInteger;
	import javax.annotation.CheckReturnValue;
	import org.junit.After;
	import org.junit.Test;
	import org.junit.runner.RunWith;
	import org.junit.runners.JUnit4;

	/**
	* Tests for DigestUtils.
	*/
	@RunWith(JUnit4.class)
	public class DigestUtilsTest {

	@After
	public void tearDown() {
	DigestUtils.configureCache(0);
	}

	private static void assertDigestCalculationConcurrency(boolean expectConcurrent,
	final boolean fastDigest, final int fileSize1, final int fileSize2,
	HashFunction hf) throws Exception {
	final CountDownLatch barrierLatch = new CountDownLatch(2); // Used to block test threads.
	final CountDownLatch readyLatch = new CountDownLatch(1); // Used to block main thread.

	FileSystem myfs = new InMemoryFileSystem(BlazeClock.instance()) {
	@Override
	protected byte[] getMD5Digest(Path path) throws IOException {
	try {
	barrierLatch.countDown();
	readyLatch.countDown();
	// Either both threads will be inside getMD5Digest at the same time or they
	// both will be blocked.
	barrierLatch.await();
	} catch (Exception e) {
	throw new IOException(e);
	}
	return super.getMD5Digest(path);
	}

	@Override
	protected byte[] getSHA1Digest(Path path) throws IOException {
	try {
	barrierLatch.countDown();
	readyLatch.countDown();
	// Either both threads will be inside getSHA1Digest at the same time or they
	// both will be blocked.
	barrierLatch.await();
	} catch (Exception e) {
	throw new IOException(e);
	}
	return super.getSHA1Digest(path);
	}

	@Override
	protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException {
	return fastDigest ? super.getDigest(path, hashFunction) : null;
	}
	};

	FileSystem.setDigestFunctionForTesting(hf);
	final Path myFile1 = myfs.getPath("/f1.dat");
	final Path myFile2 = myfs.getPath("/f2.dat");
	FileSystemUtils.writeContentAsLatin1(myFile1, Strings.repeat("a", fileSize1));
	FileSystemUtils.writeContentAsLatin1(myFile2, Strings.repeat("b", fileSize2));

	TestThread thread1 = new TestThread () {
	@Override public void runTest() throws Exception {
	DigestUtils.getDigestOrFail(myFile1, fileSize1);
	}
	};

	TestThread thread2 = new TestThread () {
	@Override public void runTest() throws Exception {
	DigestUtils.getDigestOrFail(myFile2, fileSize2);
	}
	};

	thread1.start();
	thread2.start();
	if (!expectConcurrent) { // Synchronized case.
	// Wait until at least one thread reached getMD5Digest().
	assertTrue(readyLatch.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
	// Only 1 thread should be inside getMD5Digest().
	assertEquals(1, barrierLatch.getCount());
	barrierLatch.countDown(); // Release barrier latch, allowing both threads to proceed.
	}
	// Test successful execution within 5 seconds.
	thread1.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
	thread2.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
	}

	/**
	* Ensures that MD5 calculation is synchronized for files
	* greater than 4096 bytes if MD5 is not available cheaply,
	* so machines with rotating drives don't become unusable.
	*/
	@Test
	public void testCalculationConcurrency() throws Exception {
	for (HashFunction hf : Arrays.asList(HashFunction.MD5, HashFunction.SHA1)) {
	assertDigestCalculationConcurrency(true, true, 4096, 4096, hf);
	assertDigestCalculationConcurrency(true, true, 4097, 4097, hf);
	assertDigestCalculationConcurrency(true, false, 4096, 4096, hf);
	assertDigestCalculationConcurrency(false, false, 4097, 4097, hf);
	assertDigestCalculationConcurrency(true, false, 1024, 4097, hf);
	assertDigestCalculationConcurrency(true, false, 1024, 1024, hf);
	}
	}

	public void assertRecoverFromMalformedDigest(HashFunction... hashFunctions) throws Exception {
	final byte[] malformed = {0, 0, 0};
	FileSystem myFS = new InMemoryFileSystem(BlazeClock.instance()) {
	@Override
	protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException {
	// Digest functions have more than 3 bytes, usually at least 16.
	return malformed;
	}
	};
	Path path = myFS.getPath("/file");
	FileSystemUtils.writeContentAsLatin1(path, "a");
	for (HashFunction hf : hashFunctions) {
	FileSystem.setDigestFunctionForTesting(hf);
	byte[] result = DigestUtils.getDigestOrFail(path, 1);
	assertArrayEquals(path.getDigest(), result);
	assertNotSame(malformed, result);
	assertTrue(path.isValidDigest(result));
	}
	}

	@Test
	public void testRecoverFromMalformedDigestWithoutCache() throws Exception {
	try {
	DigestUtils.getCacheStats();
	fail("Digests cache should remain disabled until configureCache is called");
	} catch (NullPointerException expected) {
	}
	assertRecoverFromMalformedDigest(HashFunction.MD5, HashFunction.SHA1);
	try {
	DigestUtils.getCacheStats();
	fail("Digests cache was unexpectedly enabled through the test");
	} catch (NullPointerException expected) {
	}
	}

	@Test
	public void testRecoverFromMalformedDigestWithCache() throws Exception {
	DigestUtils.configureCache(10);
	assertNotNull(DigestUtils.getCacheStats()); // Ensure the cache is enabled.

	// When using the cache, we cannot run our test using different hash functions because the
	// hash function is not part of the cache key. This is intentional: the hash function is
	// essentially final and can only be changed for tests. Therefore, just test the same hash
	// function twice to further exercise the cache code.
	assertRecoverFromMalformedDigest(HashFunction.MD5, HashFunction.MD5);

	assertNotNull(DigestUtils.getCacheStats()); // Ensure the cache remains enabled.
	}

	/** Helper class to assert the cache statistics. */
	private static class CacheStatsChecker {
	/** Cache statistics, grabbed at construction time. */
	private final CacheStats stats;

	private int expectedEvictionCount;
	private int expectedHitCount;
	private int expectedMissCount;

	CacheStatsChecker() {
	this.stats = DigestUtils.getCacheStats();
	}

	@CheckReturnValue
	CacheStatsChecker evictionCount(int count) {
	expectedEvictionCount = count;
	return this;
	}

	@CheckReturnValue
	CacheStatsChecker hitCount(int count) {
	expectedHitCount = count;
	return this;
	}

	@CheckReturnValue
	CacheStatsChecker missCount(int count) {
	expectedMissCount = count;
	return this;
	}

	void check() throws Exception {
	assertEquals(expectedEvictionCount, stats.evictionCount());
	assertEquals(expectedHitCount, stats.hitCount());
	assertEquals(expectedMissCount, stats.missCount());
	}
	}

	@Test
	public void testCache() throws Exception {
	final AtomicInteger getFastDigestCounter = new AtomicInteger(0);
	final AtomicInteger getDigestCounter = new AtomicInteger(0);

	FileSystem tracingFileSystem =
	new InMemoryFileSystem(BlazeClock.instance()) {
	@Override
	protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException {
	getFastDigestCounter.incrementAndGet();
	return null;
	}

	@Override
	protected byte[] getDigest(Path path) throws IOException {
	getDigestCounter.incrementAndGet();
	return super.getDigest(path);
	}
	};

	DigestUtils.configureCache(2);

	final Path file1 = tracingFileSystem.getPath("/1.txt");
	final Path file2 = tracingFileSystem.getPath("/2.txt");
	final Path file3 = tracingFileSystem.getPath("/3.txt");
	FileSystemUtils.writeContentAsLatin1(file1, "some contents");
	FileSystemUtils.writeContentAsLatin1(file2, "some other contents");
	FileSystemUtils.writeContentAsLatin1(file3, "and something else");

	byte[] digest1 = DigestUtils.getDigestOrFail(file1, file1.getFileSize());
	assertEquals(1, getFastDigestCounter.get());
	assertEquals(1, getDigestCounter.get());
	new CacheStatsChecker().evictionCount(0).hitCount(0).missCount(1).check();

	byte[] digest2 = DigestUtils.getDigestOrFail(file1, file1.getFileSize());
	assertEquals(2, getFastDigestCounter.get());
	assertEquals(1, getDigestCounter.get());
	new CacheStatsChecker().evictionCount(0).hitCount(1).missCount(1).check();

	assertArrayEquals(digest1, digest2);

	// Evict the digest for the previous file.
	DigestUtils.getDigestOrFail(file2, file2.getFileSize());
	DigestUtils.getDigestOrFail(file3, file3.getFileSize());
	new CacheStatsChecker().evictionCount(1).hitCount(1).missCount(3).check();

	// And now try to recompute it.
	byte[] digest3 = DigestUtils.getDigestOrFail(file1, file1.getFileSize());
	new CacheStatsChecker().evictionCount(2).hitCount(1).missCount(4).check();

	assertArrayEquals(digest1, digest3);
	}
	}