Revert "HBASE-14684 Try to remove all MiniMapReduceCluster in unit te…

…sts (Heng Chen)" This reverts commit ef7f846.
tongji-qqg · Oct 24, 2015 · f9efeaa · f9efeaa
1 parent 418c442
commit f9efeaa
Show file tree

Hide file tree

Showing 27 changed files with 63 additions and 4 deletions.
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
@@ -34,7 +34,6 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.client.Connection;
@@ -81,18 +80,20 @@ public class TestTableInputFormat {
   private static final Log LOG = LogFactory.getLog(TestTableInputFormat.class);
 
   private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
-
+  private static MiniMRCluster mrCluster;
   static final byte[] FAMILY = Bytes.toBytes("family");
 
   private static final byte[][] columns = new byte[][] { FAMILY };
 
   @BeforeClass
   public static void beforeClass() throws Exception {
     UTIL.startMiniCluster();
+    mrCluster = UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
+    UTIL.shutdownMiniMapReduceCluster();
     UTIL.shutdownMiniCluster();
   }
 
@@ -343,8 +344,7 @@ public void testJobConfigurableExtensionOfTableInputFormatBase() throws IOExcept
   }
 
   void testInputFormat(Class<? extends InputFormat> clazz) throws IOException {
-    Configuration conf = UTIL.getConfiguration();
-    final JobConf job = new JobConf(conf);
+    final JobConf job = MapreduceTestingShim.getJobConf(mrCluster);
     job.setInputFormat(clazz);
     job.setOutputFormat(NullOutputFormat.class);
     job.setMapperClass(ExampleVerifier.class);

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java
@@ -90,10 +90,12 @@ public class TestTableMapReduceUtil {
   public static void beforeClass() throws Exception {
     UTIL.startMiniCluster();
     presidentsTable = createAndFillTable(TableName.valueOf(TABLE_NAME));
+    UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
+    UTIL.shutdownMiniMapReduceCluster();
     UTIL.shutdownMiniCluster();
   }
 

diff --git a/...server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java b/...server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java
@@ -83,10 +83,13 @@ public static void setUpBeforeClass() throws Exception {
         TEST_UTIL.loadTable(table, INPUT_FAMILY, false);
       }
     }
+    // start MR cluster
+    TEST_UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniMapReduceCluster();
     TEST_UTIL.shutdownMiniCluster();
   }
 

diff --git a/...ver/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java b/...ver/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
@@ -154,12 +154,14 @@ public abstract void testRestoreSnapshotDoesNotCreateBackRefLinksInit(TableName
   protected void testWithMapReduce(HBaseTestingUtility util, String snapshotName,
       int numRegions, int expectedNumSplits, boolean shutdownCluster) throws Exception {
     setupCluster();
+    util.startMiniMapReduceCluster();
     try {
       Path tableDir = util.getDataTestDirOnTestFS(snapshotName);
       TableName tableName = TableName.valueOf("testWithMapReduce");
       testWithMapReduceImpl(util, tableName, snapshotName, tableDir, numRegions,
         expectedNumSplits, shutdownCluster);
     } finally {
+      util.shutdownMiniMapReduceCluster();
       tearDownCluster();
     }
   }

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java
@@ -63,12 +63,14 @@ public class TestCellCounter {
   @BeforeClass
   public static void beforeClass() throws Exception {
     UTIL.startMiniCluster();
+    UTIL.startMiniMapReduceCluster();
     FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
     FileUtil.fullyDelete(new File(OUTPUT_DIR));
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
+    UTIL.shutdownMiniMapReduceCluster();
     UTIL.shutdownMiniCluster();
   }
 

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
@@ -62,10 +62,12 @@ public class TestCopyTable {
   @BeforeClass
   public static void beforeClass() throws Exception {
     TEST_UTIL.startMiniCluster(3);
+    TEST_UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
+    TEST_UTIL.shutdownMiniMapReduceCluster();
     TEST_UTIL.shutdownMiniCluster();
   }
 

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
@@ -403,6 +403,7 @@ private void doIncrementalLoadTest(
         assertEquals("Should make 5 regions", numRegions, 5);
 
         // Generate the bulk load files
+        util.startMiniMapReduceCluster();
         runIncrementalPELoad(conf, table, r, testDir);
         // This doesn't write into the table, just makes files
         assertEquals("HFOF should not touch actual table",
@@ -472,6 +473,7 @@ private void doIncrementalLoadTest(
       assertEquals("Data should remain after reopening of regions",
           tableDigestBefore, util.checksumRows(table));
     } finally {
+      util.shutdownMiniMapReduceCluster();
       util.shutdownMiniCluster();
     }
   }
@@ -935,6 +937,7 @@ public void testExcludeAllFromMinorCompaction() throws Exception {
       // Generate two bulk load files
       conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
           true);
+      util.startMiniMapReduceCluster();
 
       try (RegionLocator regionLocator = util.getConnection().getRegionLocator(TABLE_NAME)) {
         for (int i = 0; i < 2; i++) {
@@ -975,6 +978,7 @@ public Boolean call() throws Exception {
       }, 5000);
 
     } finally {
+      util.shutdownMiniMapReduceCluster();
       util.shutdownMiniCluster();
     }
   }
@@ -1015,6 +1019,7 @@ public Boolean call() throws Exception {
       // Generate a bulk load file with more rows
       conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
           true);
+      util.startMiniMapReduceCluster();
 
       try (RegionLocator locator = util.getConnection().getRegionLocator(TABLE_NAME)) {
         runIncrementalPELoad(conf, table, locator, testDir);
@@ -1053,6 +1058,7 @@ public Boolean call() throws Exception {
       }, 5000);
 
     } finally {
+      util.shutdownMiniMapReduceCluster();
       util.shutdownMiniCluster();
     }
   }

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
@@ -430,6 +430,7 @@ private void doIncrementalLoadTest(boolean shouldChangeRegions, boolean shouldKe
       assertEquals("Should make " + regionNum + " regions", numRegions, regionNum);
 
       // Generate the bulk load files
+      util.startMiniMapReduceCluster();
       runIncrementalPELoad(conf, table.getTableDescriptor(), r, testDir);
       // This doesn't write into the table, just makes files
       assertEquals("HFOF should not touch actual table", 0, util.countRows(table));
@@ -510,6 +511,7 @@ private void doIncrementalLoadTest(boolean shouldChangeRegions, boolean shouldKe
     } finally {
       testDir.getFileSystem(conf).delete(testDir, true);
       util.deleteTable(TABLE_NAME);
+      util.shutdownMiniMapReduceCluster();
       util.shutdownMiniCluster();
     }
   }
@@ -974,6 +976,7 @@ public void testExcludeAllFromMinorCompaction() throws Exception {
       // Generate two bulk load files
       conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
           true);
+      util.startMiniMapReduceCluster();
 
       for (int i = 0; i < 2; i++) {
         Path testDir = util.getDataTestDirOnTestFS("testExcludeAllFromMinorCompaction_" + i);
@@ -1015,6 +1018,7 @@ public Boolean call() throws Exception {
       }, 5000);
 
     } finally {
+      util.shutdownMiniMapReduceCluster();
       util.shutdownMiniCluster();
     }
   }
@@ -1056,6 +1060,7 @@ public Boolean call() throws Exception {
       // Generate a bulk load file with more rows
       conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
           true);
+      util.startMiniMapReduceCluster();
 
       RegionLocator regionLocator = conn.getRegionLocator(TABLE_NAME);
       runIncrementalPELoad(conf, table.getTableDescriptor(), regionLocator, testDir);
@@ -1095,6 +1100,7 @@ public Boolean call() throws Exception {
       }, 5000);
 
     } finally {
+      util.shutdownMiniMapReduceCluster();
       util.shutdownMiniCluster();
     }
   }

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java
@@ -36,10 +36,12 @@ public class TestHRegionPartitioner {
   @BeforeClass
   public static void beforeClass() throws Exception {
     UTIL.startMiniCluster();
+    UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
+    UTIL.shutdownMiniMapReduceCluster();
     UTIL.shutdownMiniCluster();
   }
 

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java
@@ -57,10 +57,12 @@ public class TestHashTable {
   @BeforeClass
   public static void beforeClass() throws Exception {
     TEST_UTIL.startMiniCluster(3);
+    TEST_UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
+    TEST_UTIL.shutdownMiniMapReduceCluster();
     TEST_UTIL.shutdownMiniCluster();
   }
 

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -111,12 +111,14 @@ public static void beforeClass() throws Exception {
     // Up the handlers; this test needs more than usual.
     UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10);
     UTIL.startMiniCluster();
+    UTIL.startMiniMapReduceCluster();
     FQ_OUTPUT_DIR =
       new Path(OUTPUT_DIR).makeQualified(FileSystem.get(UTIL.getConfiguration())).toString();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
+    UTIL.shutdownMiniMapReduceCluster();
     UTIL.shutdownMiniCluster();
   }
 

diff --git a/...src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java b/...src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java
@@ -99,10 +99,12 @@ public static void provisionCluster() throws Exception {
     conf.set("hbase.coprocessor.master.classes", OperationAttributesTestController.class.getName());
     conf.set("hbase.coprocessor.region.classes", OperationAttributesTestController.class.getName());
     util.startMiniCluster();
+    util.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void releaseCluster() throws Exception {
+    util.shutdownMiniMapReduceCluster();
     util.shutdownMiniCluster();
   }
 

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java
@@ -90,10 +90,12 @@ public static void provisionCluster() throws Exception {
     conf.setInt("hfile.format.version", 3);
     conf.set("hbase.coprocessor.region.classes", TTLCheckingObserver.class.getName());
     util.startMiniCluster();
+    util.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void releaseCluster() throws Exception {
+    util.shutdownMiniMapReduceCluster();
     util.shutdownMiniCluster();
   }
 

diff --git a/...er/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java b/...er/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java
@@ -124,6 +124,7 @@ public static void provisionCluster() throws Exception {
     // Wait for the labels table to become available
     util.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
     createLabels();
+    util.startMiniMapReduceCluster();
   }
 
   private static void createLabels() throws IOException, InterruptedException {
@@ -147,6 +148,7 @@ public VisibilityLabelsResponse run() throws Exception {
 
   @AfterClass
   public static void releaseCluster() throws Exception {
+    util.shutdownMiniMapReduceCluster();
     util.shutdownMiniCluster();
   }
 

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
@@ -104,10 +104,12 @@ public void setConf(Configuration conf) {
   @BeforeClass
   public static void provisionCluster() throws Exception {
     util.startMiniCluster();
+    util.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void releaseCluster() throws Exception {
+    util.shutdownMiniMapReduceCluster();
     util.shutdownMiniCluster();
   }
 

diff --git a/...-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java b/...-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java
@@ -76,11 +76,13 @@ public static void beforeClass() throws Exception {
         UTIL.createMultiRegionTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY,
             OUTPUT_FAMILY });
     UTIL.loadTable(table, INPUT_FAMILY, false);
+    UTIL.startMiniMapReduceCluster();
     UTIL.waitUntilAllRegionsAssigned(MULTI_REGION_TABLE_NAME);
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
+    UTIL.shutdownMiniMapReduceCluster();
     UTIL.shutdownMiniCluster();
   }
 

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java
@@ -64,6 +64,7 @@ public class TestRowCounter {
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     TEST_UTIL.startMiniCluster();
+    TEST_UTIL.startMiniMapReduceCluster();
     Table table = TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME), Bytes.toBytes(COL_FAM));
     writeRows(table);
     table.close();
@@ -75,6 +76,7 @@ public static void setUpBeforeClass() throws Exception {
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
+    TEST_UTIL.shutdownMiniMapReduceCluster();
   }
 
   /**

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java
@@ -60,10 +60,12 @@ public class TestSyncTable {
   @BeforeClass
   public static void beforeClass() throws Exception {
     TEST_UTIL.startMiniCluster(3);
+    TEST_UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
+    TEST_UTIL.shutdownMiniMapReduceCluster();
     TEST_UTIL.shutdownMiniCluster();
   }
 

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java
@@ -84,10 +84,12 @@ public class TestTableInputFormat {
   @BeforeClass
   public static void beforeClass() throws Exception {
     UTIL.startMiniCluster();
+    mrCluster = UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
+    UTIL.shutdownMiniMapReduceCluster();
     UTIL.shutdownMiniCluster();
   }
 

diff --git a/...-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java b/...-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
@@ -84,10 +84,13 @@ public static void setUpBeforeClass() throws Exception {
     // create and fill table
     table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, INPUT_FAMILY);
     TEST_UTIL.loadTable(table, INPUT_FAMILY, false);
+    // start MR cluster
+    TEST_UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniMapReduceCluster();
     TEST_UTIL.shutdownMiniCluster();
   }
 

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java
@@ -79,10 +79,12 @@ public static void beforeClass() throws Exception {
         UTIL.createMultiRegionTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY,
             OUTPUT_FAMILY });
     UTIL.loadTable(table, INPUT_FAMILY, false);
+    UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
+    UTIL.shutdownMiniMapReduceCluster();
     UTIL.shutdownMiniCluster();
   }