    001    /*
    002     * Copyright (c) 2007-2014 Concurrent, Inc. All Rights Reserved.
    003     *
    004     * Project and contact information: http://www.cascading.org/
    005     *
    006     * This file is part of the Cascading project.
    007     *
    008     * Licensed under the Apache License, Version 2.0 (the "License");
    009     * you may not use this file except in compliance with the License.
    010     * You may obtain a copy of the License at
    011     *
    012     *     http://www.apache.org/licenses/LICENSE-2.0
    013     *
    014     * Unless required by applicable law or agreed to in writing, software
    015     * distributed under the License is distributed on an "AS IS" BASIS,
    016     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    017     * See the License for the specific language governing permissions and
    018     * limitations under the License.
    019     */
    020
    021    package cascading.platform;
    022
    023    import java.io.IOException;
    024    import java.util.Comparator;
    025    import java.util.HashMap;
    026    import java.util.Map;
    027
    028    import cascading.flow.FlowConnector;
    029    import cascading.flow.FlowProcess;
    030    import cascading.scheme.Scheme;
    031    import cascading.scheme.util.FieldTypeResolver;
    032    import cascading.tap.SinkMode;
    033    import cascading.tap.Tap;
    034    import cascading.tap.partition.Partition;
    035    import cascading.tuple.Fields;
    036    import org.slf4j.Logger;
    037    import org.slf4j.LoggerFactory;
    038
    039    /**
    040     *
    041     */
    042    public abstract class TestPlatform
    043      {
    044      private static final Logger LOG = LoggerFactory.getLogger( TestPlatform.class );
    045
    046      public static final String CLUSTER_TESTING_PROPERTY = "test.cluster.enabled";
    047
    048      private boolean useCluster = false;
    049      private boolean enableCluster = true;
    050      private int numMappers = 0;
    051      private int numReducers = 0;
    052
    053      /**
    054       * Method getGlobalProperties fetches all "platform." prefixed system properties.
    055       * <p/>
    056       * Sub-classes of TestPlatform should use these values as overrides before returning from
    057       * {@link #getProperties()}.
    058       *
    059       * @return a Map of properties
    060       */
    061      public static Map<Object, Object> getGlobalProperties()
    062        {
    063        HashMap<Object, Object> properties = new HashMap<Object, Object>();
    064
    065        for( String propertyName : System.getProperties().stringPropertyNames() )
    066          {
    067          if( propertyName.startsWith( "platform." ) )
    068            properties.put( propertyName.substring( "platform.".length() ), System.getProperty( propertyName ) );
    069          }
    070
    071        if( !properties.isEmpty() )
    072          LOG.info( "platform property overrides: ", properties );
    073
    074        return properties;
    075        }
    076
    077      protected TestPlatform()
    078        {
    079        enableCluster = Boolean.parseBoolean( System.getProperty( CLUSTER_TESTING_PROPERTY, Boolean.toString( enableCluster ) ) );
    080        }
    081
    082      public String getName()
    083        {
    084        return getClass().getSimpleName().replaceAll( "^(.*)Platform$", "$1" ).toLowerCase();
    085        }
    086
    087      public boolean isMapReduce()
    088        {
    089        return false;
    090        }
    091
    092      public int getNumMappers()
    093        {
    094        return numMappers;
    095        }
    096
    097      public void setNumMappers( int numMappers )
    098        {
    099        this.numMappers = numMappers;
    100        }
    101
    102      public int getNumReducers()
    103        {
    104        return numReducers;
    105        }
    106
    107      public void setNumReducers( int numReducers )
    108        {
    109        this.numReducers = numReducers;
    110        }
    111
    112      public void setNumMapTasks( Map<Object, Object> properties, int numMapTasks )
    113        {
    114        // do nothing
    115        }
    116
    117      public void setNumReduceTasks( Map<Object, Object> properties, int numReduceTasks )
    118        {
    119        // do nothing
    120        }
    121
    122      public Integer getNumMapTasks( Map<Object, Object> properties )
    123        {
    124        return null;
    125        }
    126
    127      public Integer getNumReduceTasks( Map<Object, Object> properties )
    128        {
    129        return null;
    130        }
    131
    132      public abstract void setUp() throws IOException;
    133
    134      public abstract Map<Object, Object> getProperties();
    135
    136      public abstract void tearDown();
    137
    138      public void setUseCluster( boolean useCluster )
    139        {
    140        this.useCluster = useCluster;
    141        }
    142
    143      public boolean isUseCluster()
    144        {
    145        return enableCluster && useCluster;
    146        }
    147
    148      public abstract void copyFromLocal( String inputFile ) throws IOException;
    149
    150      public abstract void copyToLocal( String outputFile ) throws IOException;
    151
    152      public abstract boolean remoteExists( String outputFile ) throws IOException;
    153
    154      public abstract boolean remoteRemove( String outputFile, boolean recursive ) throws IOException;
    155
    156      public abstract FlowProcess getFlowProcess();
    157
    158      public abstract FlowConnector getFlowConnector( Map<Object, Object> properties );
    159
    160      public FlowConnector getFlowConnector()
    161        {
    162        return getFlowConnector( getProperties() );
    163        }
    164
    165      public abstract Tap getTap( Scheme scheme, String filename, SinkMode mode );
    166
    167      public Tap getTextFile( Fields sourceFields, String filename )
    168        {
    169        return getTextFile( sourceFields, filename, SinkMode.KEEP );
    170        }
    171
    172      public Tap getTextFile( String filename )
    173        {
    174        return getTextFile( filename, SinkMode.KEEP );
    175        }
    176
    177      public Tap getTextFile( String filename, SinkMode mode )
    178        {
    179        return getTextFile( null, filename, mode );
    180        }
    181
    182      public Tap getTextFile( Fields sourceFields, String filename, SinkMode mode )
    183        {
    184        return getTextFile( sourceFields, Fields.ALL, filename, mode );
    185        }
    186
    187      public abstract Tap getTextFile( Fields sourceFields, Fields sinkFields, String filename, SinkMode mode );
    188
    189      public Tap getDelimitedFile( Fields fields, String delimiter, String filename )
    190        {
    191        return getDelimitedFile( fields, false, delimiter, "\"", null, filename, SinkMode.KEEP );
    192        }
    193
    194      public Tap getDelimitedFile( Fields fields, String delimiter, String filename, SinkMode mode )
    195        {
    196        return getDelimitedFile( fields, false, delimiter, "\"", null, filename, mode );
    197        }
    198
    199      @Deprecated
    200      public Tap getDelimitedFile( Fields fields, String filename, SinkMode mode )
    201        {
    202        return getDelimitedFile( fields, false, "\t", "\"", null, filename, mode );
    203        }
    204
    205      @Deprecated
    206      public Tap getDelimitedFile( Fields fields, boolean hasHeader, String filename, SinkMode mode )
    207        {
    208        return getDelimitedFile( fields, hasHeader, "\t", "\"", null, filename, mode );
    209        }
    210
    211      public Tap getTabDelimitedFile( Fields fields, String filename, SinkMode mode )
    212        {
    213        return getDelimitedFile( fields, false, "\t", "\"", null, filename, mode );
    214        }
    215
    216      public Tap getTabDelimitedFile( Fields fields, boolean hasHeader, String filename, SinkMode mode )
    217        {
    218        return getDelimitedFile( fields, hasHeader, "\t", "\"", null, filename, mode );
    219        }
    220
    221      public Tap getDelimitedFile( Fields fields, boolean hasHeader, String delimiter, String quote, String filename, SinkMode mode )
    222        {
    223        return getDelimitedFile( fields, hasHeader, delimiter, quote, null, filename, mode );
    224        }
    225
    226      public Tap getDelimitedFile( Fields fields, String delimiter, String quote, String filename, SinkMode mode )
    227        {
    228        return getDelimitedFile( fields, false, delimiter, quote, null, filename, mode );
    229        }
    230
    231      public Tap getDelimitedFile( Fields fields, String delimiter, Class[] types, String filename, SinkMode mode )
    232        {
    233        return getDelimitedFile( fields, false, delimiter, "\"", types, filename, mode );
    234        }
    235
    236      public abstract Tap getDelimitedFile( Fields fields, boolean hasHeader, String delimiter, String quote, Class[] types, String filename, SinkMode mode );
    237
    238      public abstract Tap getDelimitedFile( Fields fields, boolean skipHeader, boolean writeHeader, String delimiter, String quote, Class[] types, String filename, SinkMode mode );
    239
    240      public abstract Tap getDelimitedFile( String delimiter, String quote, FieldTypeResolver fieldTypeResolver, String filename, SinkMode mode );
    241
    242      public abstract Tap getTemplateTap( Tap sink, String pathTemplate, int openThreshold );
    243
    244      public abstract Tap getTemplateTap( Tap sink, String pathTemplate, Fields fields, int openThreshold );
    245
    246      public abstract Tap getPartitionTap( Tap sink, Partition partition, int openThreshold );
    247
    248      public abstract Scheme getTestConfigDefScheme();
    249
    250      public abstract Scheme getTestFailScheme();
    251
    252      public abstract Comparator getLongComparator( boolean reverseSort );
    253
    254      public abstract Comparator getStringComparator( boolean reverseSort );
    255
    256      public abstract String getHiddenTemporaryPath();
    257      }