cascading.platform.hadoop
Class HadoopPlatform
java.lang.Object
cascading.platform.TestPlatform
cascading.platform.hadoop.HadoopPlatform
public class HadoopPlatform
- extends TestPlatform
Class HadoopPlatform is automatically loaded and injected into a PlatformTestCase
instance
so that all *PlatformTest classes can be tested against Apache Hadoop.
This platform works in three modes.
Hadoop standalone mode is when Hadoop is NOT run as a cluster, and all
child tasks are in process and in memory of the "client" side code.
Hadoop mini cluster mode where a cluster is created on demand using the Hadoop MiniDFSCluster and MiniMRCluster
utilities. When a PlatformTestCase requests to use a cluster, this is the default cluster. All properties are
pulled from the current CLASSPATH via the JobConf.
Lastly remote cluster mode is enabled when the System property "mapred.jar" is set. This is a Hadoop property
specifying the Hadoop "job jar" to be used cluster side. This MUST be the Cascading test suite and dependencies
packaged in a Hadoop compatible way. This is left to be implemented by the framework using this mode. Additionally
these properties may optionally be set if not already in the CLASSPATH; fs.default.name and mapred.job.tracker.
Method Summary |
void |
copyFromLocal(String inputFile)
|
void |
copyToLocal(String outputFile)
|
Tap |
getDelimitedFile(Fields fields,
boolean skipHeader,
boolean writeHeader,
String delimiter,
String quote,
Class[] types,
String filename,
SinkMode mode)
|
Tap |
getDelimitedFile(Fields fields,
boolean hasHeader,
String delimiter,
String quote,
Class[] types,
String filename,
SinkMode mode)
|
Tap |
getDelimitedFile(String delimiter,
String quote,
FieldTypeResolver fieldTypeResolver,
String filename,
SinkMode mode)
|
FlowConnector |
getFlowConnector(Map<Object,Object> properties)
|
FlowProcess |
getFlowProcess()
|
String |
getHiddenTemporaryPath()
|
JobConf |
getJobConf()
|
Comparator |
getLongComparator(boolean reverseSort)
|
Map<Object,Object> |
getProperties()
|
Comparator |
getStringComparator(boolean reverseSort)
|
Tap |
getTap(Scheme scheme,
String filename,
SinkMode mode)
|
Tap |
getTemplateTap(Tap sink,
String pathTemplate,
Fields fields,
int openThreshold)
|
Tap |
getTemplateTap(Tap sink,
String pathTemplate,
int openThreshold)
|
Scheme |
getTestConfigDefScheme()
|
Scheme |
getTestFailScheme()
|
Tap |
getTextFile(Fields sourceFields,
Fields sinkFields,
String filename,
SinkMode mode)
|
boolean |
isHDFSAvailable()
|
boolean |
isMapReduce()
|
boolean |
remoteExists(String outputFile)
|
boolean |
remoteRemove(String outputFile,
boolean recursive)
|
void |
setNumMapTasks(int numMapTasks)
|
void |
setNumReduceTasks(int numReduceTasks)
|
void |
setUp()
|
void |
tearDown()
|
Methods inherited from class cascading.platform.TestPlatform |
getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getFlowConnector, getGlobalProperties, getName, getNumMappers, getNumReducers, getTabDelimitedFile, getTabDelimitedFile, getTextFile, getTextFile, getTextFile, getTextFile, isUseCluster, setNumMappers, setNumReducers, setUseCluster |
Methods inherited from class java.lang.Object |
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
dfs
public static transient org.apache.hadoop.hdfs.MiniDFSCluster dfs
fileSys
public static transient FileSystem fileSys
mr
public static transient MiniMRCluster mr
jobConf
public static transient JobConf jobConf
properties
public static transient Map<Object,Object> properties
numMapTasks
public int numMapTasks
numReduceTasks
public int numReduceTasks
HadoopPlatform
public HadoopPlatform()
isMapReduce
public boolean isMapReduce()
- Overrides:
isMapReduce
in class TestPlatform
setNumMapTasks
public void setNumMapTasks(int numMapTasks)
setNumReduceTasks
public void setNumReduceTasks(int numReduceTasks)
setUp
public void setUp()
throws IOException
- Specified by:
setUp
in class TestPlatform
- Throws:
IOException
getProperties
public Map<Object,Object> getProperties()
- Specified by:
getProperties
in class TestPlatform
tearDown
public void tearDown()
- Specified by:
tearDown
in class TestPlatform
getJobConf
public JobConf getJobConf()
isHDFSAvailable
public boolean isHDFSAvailable()
getFlowConnector
public FlowConnector getFlowConnector(Map<Object,Object> properties)
- Specified by:
getFlowConnector
in class TestPlatform
getFlowProcess
public FlowProcess getFlowProcess()
- Specified by:
getFlowProcess
in class TestPlatform
copyFromLocal
public void copyFromLocal(String inputFile)
throws IOException
- Specified by:
copyFromLocal
in class TestPlatform
- Throws:
IOException
copyToLocal
public void copyToLocal(String outputFile)
throws IOException
- Specified by:
copyToLocal
in class TestPlatform
- Throws:
IOException
remoteExists
public boolean remoteExists(String outputFile)
throws IOException
- Specified by:
remoteExists
in class TestPlatform
- Throws:
IOException
remoteRemove
public boolean remoteRemove(String outputFile,
boolean recursive)
throws IOException
- Specified by:
remoteRemove
in class TestPlatform
- Throws:
IOException
getTap
public Tap getTap(Scheme scheme,
String filename,
SinkMode mode)
- Specified by:
getTap
in class TestPlatform
getTextFile
public Tap getTextFile(Fields sourceFields,
Fields sinkFields,
String filename,
SinkMode mode)
- Specified by:
getTextFile
in class TestPlatform
getDelimitedFile
public Tap getDelimitedFile(Fields fields,
boolean hasHeader,
String delimiter,
String quote,
Class[] types,
String filename,
SinkMode mode)
- Specified by:
getDelimitedFile
in class TestPlatform
getDelimitedFile
public Tap getDelimitedFile(Fields fields,
boolean skipHeader,
boolean writeHeader,
String delimiter,
String quote,
Class[] types,
String filename,
SinkMode mode)
- Specified by:
getDelimitedFile
in class TestPlatform
getDelimitedFile
public Tap getDelimitedFile(String delimiter,
String quote,
FieldTypeResolver fieldTypeResolver,
String filename,
SinkMode mode)
- Specified by:
getDelimitedFile
in class TestPlatform
getTemplateTap
public Tap getTemplateTap(Tap sink,
String pathTemplate,
int openThreshold)
- Specified by:
getTemplateTap
in class TestPlatform
getTemplateTap
public Tap getTemplateTap(Tap sink,
String pathTemplate,
Fields fields,
int openThreshold)
- Specified by:
getTemplateTap
in class TestPlatform
getTestConfigDefScheme
public Scheme getTestConfigDefScheme()
- Specified by:
getTestConfigDefScheme
in class TestPlatform
getTestFailScheme
public Scheme getTestFailScheme()
- Specified by:
getTestFailScheme
in class TestPlatform
getLongComparator
public Comparator getLongComparator(boolean reverseSort)
- Specified by:
getLongComparator
in class TestPlatform
getStringComparator
public Comparator getStringComparator(boolean reverseSort)
- Specified by:
getStringComparator
in class TestPlatform
getHiddenTemporaryPath
public String getHiddenTemporaryPath()
- Specified by:
getHiddenTemporaryPath
in class TestPlatform
Copyright © 2007-2013 Concurrent, Inc. All Rights Reserved.