    001    /*
    002     * Copyright (c) 2007-2014 Concurrent, Inc. All Rights Reserved.
    003     *
    004     * Project and contact information: http://www.cascading.org/
    005     *
    006     * This file is part of the Cascading project.
    007     *
    008     * Licensed under the Apache License, Version 2.0 (the "License");
    009     * you may not use this file except in compliance with the License.
    010     * You may obtain a copy of the License at
    011     *
    012     *     http://www.apache.org/licenses/LICENSE-2.0
    013     *
    014     * Unless required by applicable law or agreed to in writing, software
    015     * distributed under the License is distributed on an "AS IS" BASIS,
    016     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    017     * See the License for the specific language governing permissions and
    018     * limitations under the License.
    019     */
    020
    021    package cascading.tap.hadoop.util;
    022
    023    import java.io.IOException;
    024    import java.util.Set;
    025
    026    import cascading.CascadingException;
    027    import cascading.flow.FlowProcess;
    028    import cascading.flow.planner.Scope;
    029    import cascading.scheme.Scheme;
    030    import cascading.scheme.SinkCall;
    031    import cascading.scheme.SourceCall;
    032    import cascading.scheme.hadoop.SequenceFile;
    033    import cascading.tap.Tap;
    034    import cascading.tap.hadoop.Hfs;
    035    import cascading.tuple.Fields;
    036    import cascading.tuple.Tuple;
    037    import org.apache.hadoop.fs.Path;
    038    import org.apache.hadoop.mapred.JobConf;
    039    import org.apache.hadoop.mapred.OutputCollector;
    040    import org.apache.hadoop.mapred.RecordReader;
    041    import org.apache.hadoop.mapred.lib.NullOutputFormat;
    042
    043    /** Class TempHfs creates a temporary {@link cascading.tap.Tap} instance for use internally. */
    044    public class TempHfs extends Hfs
    045      {
    046      /** Field name */
    047      final String name;
    048      /** Field schemeClass */
    049      private Class<? extends Scheme> schemeClass;
    050      /** Field temporaryPath */
    051
    052      /** Class NullScheme is a noop scheme used as a placeholder */
    053      private static class NullScheme extends Scheme<JobConf, RecordReader, OutputCollector, Object, Object>
    054        {
    055        @Override
    056        public void sourceConfInit( FlowProcess<JobConf> flowProcess, Tap<JobConf, RecordReader, OutputCollector> tap, JobConf conf )
    057          {
    058          // do nothing
    059          }
    060
    061        @Override
    062        public void sinkConfInit( FlowProcess<JobConf> flowProcess, Tap<JobConf, RecordReader, OutputCollector> tap, JobConf conf )
    063          {
    064          conf.setOutputKeyClass( Tuple.class );
    065          conf.setOutputValueClass( Tuple.class );
    066          conf.setOutputFormat( NullOutputFormat.class );
    067          }
    068
    069        @Override
    070        public boolean source( FlowProcess<JobConf> flowProcess, SourceCall<Object, RecordReader> sourceCall ) throws IOException
    071          {
    072          return false;
    073          }
    074
    075        @Override
    076        public void sink( FlowProcess<JobConf> flowProcess, SinkCall<Object, OutputCollector> sinkCall ) throws IOException
    077          {
    078          }
    079        }
    080
    081      /**
    082       * Constructor TempHfs creates a new TempHfs instance.
    083       *
    084       * @param name   of type String
    085       * @param isNull of type boolean
    086       */
    087      public TempHfs( JobConf conf, String name, boolean isNull )
    088        {
    089        super( isNull ? new NullScheme() : new SequenceFile()
    090        {
    091        } );
    092        this.name = name;
    093        this.stringPath = initTemporaryPath( conf, true );
    094        }
    095
    096      /**
    097       * Constructor TempDfs creates a new TempDfs instance.
    098       *
    099       * @param name of type String
    100       */
    101      public TempHfs( JobConf conf, String name, Class<? extends Scheme> schemeClass )
    102        {
    103        this( conf, name, schemeClass, true );
    104        }
    105
    106      public TempHfs( JobConf conf, String name, Class<? extends Scheme> schemeClass, boolean unique )
    107        {
    108        this.name = name;
    109
    110        if( schemeClass == null )
    111          this.schemeClass = SequenceFile.class;
    112        else
    113          this.schemeClass = schemeClass;
    114
    115        this.stringPath = initTemporaryPath( conf, unique );
    116        }
    117
    118      public Class<? extends Scheme> getSchemeClass()
    119        {
    120        return schemeClass;
    121        }
    122
    123      private String initTemporaryPath( JobConf conf, boolean unique )
    124        {
    125        String child = unique ? makeTemporaryPathDirString( name ) : name;
    126
    127        return new Path( getTempPath( conf ), child ).toString();
    128        }
    129
    130      @Override
    131      public Scope outgoingScopeFor( Set<Scope> incomingScopes )
    132        {
    133        Fields fields = incomingScopes.iterator().next().getIncomingTapFields();
    134
    135        setSchemeUsing( fields );
    136
    137        return new Scope( fields );
    138        }
    139
    140      private void setSchemeUsing( Fields fields )
    141        {
    142        try
    143          {
    144          setScheme( schemeClass.getConstructor( Fields.class ).newInstance( fields ) );
    145          }
    146        catch( Exception exception )
    147          {
    148          throw new CascadingException( "unable to create specified scheme: " + schemeClass.getName(), exception );
    149          }
    150        }
    151
    152      @Override
    153      public boolean isTemporary()
    154        {
    155        return true;
    156        }
    157
    158      @Override
    159      public String toString()
    160        {
    161        return getClass().getSimpleName() + "[\"" + getScheme() + "\"]" + "[" + name + "]";
    162        }
    163
    164      @Override
    165      public boolean equals( Object object )
    166        {
    167        // we are only overriding since the path is lazily initialized.
    168
    169        if( this == object )
    170          return true;
    171        if( object == null || getClass() != object.getClass() )
    172          return false;
    173        if( !super.equals( object ) )
    174          return false;
    175
    176        TempHfs tempHfs = (TempHfs) object;
    177
    178        if( name != null ? !name.equals( tempHfs.name ) : tempHfs.name != null )
    179          return false;
    180
    181        return true;
    182        }
    183
    184      @Override
    185      public int hashCode()
    186        {
    187        // don't use super hashCode() as path changes during runtime
    188        return 31 * ( System.identityHashCode( this ) + name != null ? name.hashCode() : 0 );
    189        }
    190      }