cascading.platform.hadoop
Class HadoopPlatform

java.lang.Object
  extended by cascading.platform.TestPlatform
      extended by cascading.platform.hadoop.HadoopPlatform

public class HadoopPlatform
extends TestPlatform

Class HadoopPlatform is automatically loaded and injected into a PlatformTestCase instance so that all *PlatformTest classes can be tested against Apache Hadoop.

This platform works in three modes.

Hadoop standalone mode is when Hadoop is NOT run as a cluster, and all child tasks are in process and in memory of the "client" side code.

Hadoop mini cluster mode where a cluster is created on demand using the Hadoop MiniDFSCluster and MiniMRCluster utilities. When a PlatformTestCase requests to use a cluster, this is the default cluster. All properties are pulled from the current CLASSPATH via the JobConf.

Lastly remote cluster mode is enabled when the System property "mapred.jar" is set. This is a Hadoop property specifying the Hadoop "job jar" to be used cluster side. This MUST be the Cascading test suite and dependencies packaged in a Hadoop compatible way. This is left to be implemented by the framework using this mode. Additionally these properties may optionally be set if not already in the CLASSPATH; fs.default.name and mapred.job.tracker.


Field Summary
static org.apache.hadoop.hdfs.MiniDFSCluster dfs
           
static FileSystem fileSys
           
static JobConf jobConf
           
static MiniMRCluster mr
           
 int numMapTasks
           
 int numReduceTasks
           
static Map<Object,Object> properties
           
 
Fields inherited from class cascading.platform.TestPlatform
CLUSTER_TESTING_PROPERTY
 
Constructor Summary
HadoopPlatform()
           
 
Method Summary
 void copyFromLocal(String inputFile)
           
 void copyToLocal(String outputFile)
           
 Tap getDelimitedFile(Fields fields, boolean skipHeader, boolean writeHeader, String delimiter, String quote, Class[] types, String filename, SinkMode mode)
           
 Tap getDelimitedFile(Fields fields, boolean hasHeader, String delimiter, String quote, Class[] types, String filename, SinkMode mode)
           
 Tap getDelimitedFile(String delimiter, String quote, FieldTypeResolver fieldTypeResolver, String filename, SinkMode mode)
           
 FlowConnector getFlowConnector(Map<Object,Object> properties)
           
 FlowProcess getFlowProcess()
           
 String getHiddenTemporaryPath()
           
 JobConf getJobConf()
           
 Comparator getLongComparator(boolean reverseSort)
           
 Map<Object,Object> getProperties()
           
 Comparator getStringComparator(boolean reverseSort)
           
 Tap getTap(Scheme scheme, String filename, SinkMode mode)
           
 Tap getTemplateTap(Tap sink, String pathTemplate, Fields fields, int openThreshold)
           
 Tap getTemplateTap(Tap sink, String pathTemplate, int openThreshold)
           
 Scheme getTestConfigDefScheme()
           
 Scheme getTestFailScheme()
           
 Tap getTextFile(Fields sourceFields, Fields sinkFields, String filename, SinkMode mode)
           
 boolean isHDFSAvailable()
           
 boolean isMapReduce()
           
 boolean remoteExists(String outputFile)
           
 boolean remoteRemove(String outputFile, boolean recursive)
           
 void setNumMapTasks(int numMapTasks)
           
 void setNumReduceTasks(int numReduceTasks)
           
 void setUp()
           
 void tearDown()
           
 
Methods inherited from class cascading.platform.TestPlatform
getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getFlowConnector, getGlobalProperties, getName, getNumMappers, getNumReducers, getTabDelimitedFile, getTabDelimitedFile, getTextFile, getTextFile, getTextFile, getTextFile, isUseCluster, setNumMappers, setNumReducers, setUseCluster
 
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
 

Field Detail

dfs

public static transient org.apache.hadoop.hdfs.MiniDFSCluster dfs

fileSys

public static transient FileSystem fileSys

mr

public static transient MiniMRCluster mr

jobConf

public static transient JobConf jobConf

properties

public static transient Map<Object,Object> properties

numMapTasks

public int numMapTasks

numReduceTasks

public int numReduceTasks
Constructor Detail

HadoopPlatform

public HadoopPlatform()
Method Detail

isMapReduce

public boolean isMapReduce()
Overrides:
isMapReduce in class TestPlatform

setNumMapTasks

public void setNumMapTasks(int numMapTasks)

setNumReduceTasks

public void setNumReduceTasks(int numReduceTasks)

setUp

public void setUp()
           throws IOException
Specified by:
setUp in class TestPlatform
Throws:
IOException

getProperties

public Map<Object,Object> getProperties()
Specified by:
getProperties in class TestPlatform

tearDown

public void tearDown()
Specified by:
tearDown in class TestPlatform

getJobConf

public JobConf getJobConf()

isHDFSAvailable

public boolean isHDFSAvailable()

getFlowConnector

public FlowConnector getFlowConnector(Map<Object,Object> properties)
Specified by:
getFlowConnector in class TestPlatform

getFlowProcess

public FlowProcess getFlowProcess()
Specified by:
getFlowProcess in class TestPlatform

copyFromLocal

public void copyFromLocal(String inputFile)
                   throws IOException
Specified by:
copyFromLocal in class TestPlatform
Throws:
IOException

copyToLocal

public void copyToLocal(String outputFile)
                 throws IOException
Specified by:
copyToLocal in class TestPlatform
Throws:
IOException

remoteExists

public boolean remoteExists(String outputFile)
                     throws IOException
Specified by:
remoteExists in class TestPlatform
Throws:
IOException

remoteRemove

public boolean remoteRemove(String outputFile,
                            boolean recursive)
                     throws IOException
Specified by:
remoteRemove in class TestPlatform
Throws:
IOException

getTap

public Tap getTap(Scheme scheme,
                  String filename,
                  SinkMode mode)
Specified by:
getTap in class TestPlatform

getTextFile

public Tap getTextFile(Fields sourceFields,
                       Fields sinkFields,
                       String filename,
                       SinkMode mode)
Specified by:
getTextFile in class TestPlatform

getDelimitedFile

public Tap getDelimitedFile(Fields fields,
                            boolean hasHeader,
                            String delimiter,
                            String quote,
                            Class[] types,
                            String filename,
                            SinkMode mode)
Specified by:
getDelimitedFile in class TestPlatform

getDelimitedFile

public Tap getDelimitedFile(Fields fields,
                            boolean skipHeader,
                            boolean writeHeader,
                            String delimiter,
                            String quote,
                            Class[] types,
                            String filename,
                            SinkMode mode)
Specified by:
getDelimitedFile in class TestPlatform

getDelimitedFile

public Tap getDelimitedFile(String delimiter,
                            String quote,
                            FieldTypeResolver fieldTypeResolver,
                            String filename,
                            SinkMode mode)
Specified by:
getDelimitedFile in class TestPlatform

getTemplateTap

public Tap getTemplateTap(Tap sink,
                          String pathTemplate,
                          int openThreshold)
Specified by:
getTemplateTap in class TestPlatform

getTemplateTap

public Tap getTemplateTap(Tap sink,
                          String pathTemplate,
                          Fields fields,
                          int openThreshold)
Specified by:
getTemplateTap in class TestPlatform

getTestConfigDefScheme

public Scheme getTestConfigDefScheme()
Specified by:
getTestConfigDefScheme in class TestPlatform

getTestFailScheme

public Scheme getTestFailScheme()
Specified by:
getTestFailScheme in class TestPlatform

getLongComparator

public Comparator getLongComparator(boolean reverseSort)
Specified by:
getLongComparator in class TestPlatform

getStringComparator

public Comparator getStringComparator(boolean reverseSort)
Specified by:
getStringComparator in class TestPlatform

getHiddenTemporaryPath

public String getHiddenTemporaryPath()
Specified by:
getHiddenTemporaryPath in class TestPlatform


Copyright © 2007-2013 Concurrent, Inc. All Rights Reserved.