    001    /*
    002     * Copyright (c) 2007-2014 Concurrent, Inc. All Rights Reserved.
    003     *
    004     * Project and contact information: http://www.cascading.org/
    005     *
    006     * This file is part of the Cascading project.
    007     *
    008     * Licensed under the Apache License, Version 2.0 (the "License");
    009     * you may not use this file except in compliance with the License.
    010     * You may obtain a copy of the License at
    011     *
    012     *     http://www.apache.org/licenses/LICENSE-2.0
    013     *
    014     * Unless required by applicable law or agreed to in writing, software
    015     * distributed under the License is distributed on an "AS IS" BASIS,
    016     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    017     * See the License for the specific language governing permissions and
    018     * limitations under the License.
    019     */
    020
    021    package cascading;
    022
    023    import java.io.File;
    024    import java.io.IOException;
    025    import java.util.Collection;
    026    import java.util.Collections;
    027    import java.util.LinkedList;
    028    import java.util.List;
    029    import java.util.ListIterator;
    030
    031    import junit.framework.Test;
    032    import junit.framework.TestCase;
    033    import junit.framework.TestSuite;
    034    import org.apache.commons.io.FileUtils;
    035    import org.apache.commons.io.LineIterator;
    036    import org.apache.commons.io.filefilter.RegexFileFilter;
    037    import org.apache.commons.io.filefilter.TrueFileFilter;
    038    import org.slf4j.Logger;
    039    import org.slf4j.LoggerFactory;
    040
    041    /**
    042     *
    043     */
    044    public class ComparePlatformsTest extends CascadingTestCase
    045      {
    046      private static final Logger LOG = LoggerFactory.getLogger( ComparePlatformsTest.class );
    047      public static final String NONDETERMINISTIC = "-nondeterministic";
    048
    049      public static Test suite() throws Exception
    050        {
    051        String root = System.getProperty( "test.output.roots" );
    052
    053        LOG.info( "output roots: {}", root );
    054
    055        String[] roots = root.split( "," );
    056
    057        File localRoot = new File( find( roots, "/cascading-local/" ), "local" );
    058        File hadoopRoot = new File( find( roots, "/cascading-hadoop/" ), "hadoop" );
    059        File hadoop2Root = new File( find( roots, "/cascading-hadoop2-mr1/" ), "hadoop2-mr1" );
    060
    061        LOG.info( "local path: {}", localRoot );
    062        LOG.info( "hadoop path: {}", hadoopRoot );
    063        LOG.info( "hadoop2 path: {}", hadoop2Root );
    064
    065        TestSuite suite = new TestSuite();
    066
    067        createComparisons( "local~hadoop", localRoot, hadoopRoot, suite );
    068        createComparisons( "local~hadoop2-mr1", localRoot, hadoop2Root, suite );
    069
    070        return suite;
    071        }
    072
    073      private static void createComparisons( String comparison, File lhsRoot, File rhsRoot, TestSuite suite )
    074        {
    075        LOG.info( "comparing directory: {}, with: {}", lhsRoot, rhsRoot );
    076
    077        LinkedList<File> lhsFiles = new LinkedList<File>( FileUtils.listFiles( lhsRoot, new RegexFileFilter( "^[\\w-]+" ), TrueFileFilter.INSTANCE ) );
    078        LinkedList<File> rhsFiles = new LinkedList<File>();
    079
    080        LOG.info( "found lhs files: {}", lhsFiles.size() );
    081
    082        int rootLength = lhsRoot.toString().length() + 1;
    083
    084        ListIterator<File> iterator = lhsFiles.listIterator();
    085        while( iterator.hasNext() )
    086          {
    087          File localFile = iterator.next();
    088          File file = new File( rhsRoot, localFile.toString().substring( rootLength ) );
    089
    090          if( localFile.toString().endsWith( NONDETERMINISTIC ) )
    091            iterator.remove();
    092          else if( file.exists() )
    093            rhsFiles.add( file );
    094          else
    095            iterator.remove();
    096          }
    097
    098        LOG.info( "running {} comparisons", lhsFiles.size() );
    099
    100        for( int i = 0; i < lhsFiles.size(); i++ )
    101          {
    102          File localFile = lhsFiles.get( i );
    103          File hadoopFile = rhsFiles.get( i );
    104
    105          suite.addTest( new CompareTestCase( comparison, localFile, hadoopFile ) );
    106          }
    107        }
    108
    109      private static String find( String[] roots, String string )
    110        {
    111        for( String root : roots )
    112          {
    113          if( root.contains( string ) )
    114            return root;
    115          }
    116
    117        throw new IllegalStateException( "not found in roots: " + string );
    118        }
    119
    120      public static class CompareTestCase extends TestCase
    121        {
    122        File localFile;
    123        File hadoopFile;
    124
    125        public CompareTestCase( String comparison, File localFile, File hadoopFile )
    126          {
    127          super( "testFiles" );
    128
    129          this.localFile = localFile;
    130          this.hadoopFile = hadoopFile;
    131
    132          // craps out junit, unsure how to set display name
    133    //      setName( String.format( "%s..%s", comparison, localFile.getName() ) ); // relevant bits have same file name
    134          }
    135
    136        @org.junit.Test
    137        public void testFiles() throws IOException
    138          {
    139          LinkedList<String> localLines = getLines( localFile );
    140          LinkedList<String> hadoopLines = getLines( hadoopFile );
    141
    142          assertEquals( localFile + " != " + hadoopFile, localLines.size(), hadoopLines.size() );
    143
    144          if( localLines.size() == 0 )
    145            return;
    146
    147          Collections.sort( localLines );
    148          Collections.sort( hadoopLines );
    149
    150          if( hasLineNumbers( localLines ) )
    151            {
    152            trimLineNumbers( localLines );
    153            trimLineNumbers( hadoopLines );
    154            }
    155
    156          for( int i = 0; i < localLines.size(); i++ )
    157            {
    158            String localLine = localLines.get( i );
    159
    160            assertTrue( localFile + " - not in hadoop lines: " + localLine, hadoopLines.contains( localLine ) );
    161            }
    162          }
    163
    164        private void trimLineNumbers( LinkedList<String> lines )
    165          {
    166          ListIterator<String> iterator = lines.listIterator();
    167
    168          while( iterator.hasNext() )
    169            iterator.set( iterator.next().replaceFirst( "^\\d+\\s(.*)$", "$1" ) );
    170          }
    171
    172        private boolean hasLineNumbers( List<String> lines )
    173          {
    174          List<Integer> values = new LinkedList<Integer>();
    175
    176          for( String line : lines )
    177            {
    178            if( !line.matches( "^\\d+\\s.*$" ) )
    179              return false;
    180
    181            String value = line.replaceFirst( "^(\\d+)\\s.*$", "$1" );
    182
    183            if( value == null || value.isEmpty() )
    184              return false;
    185
    186            values.add( Integer.parseInt( value ) );
    187            }
    188
    189          Collections.sort( values );
    190
    191          int last = -1;
    192          for( Integer value : values )
    193            {
    194            if( last >= value )
    195              return false;
    196
    197            last = value;
    198            }
    199
    200          return true;
    201          }
    202
    203        private LinkedList<String> getLines( File localFile ) throws IOException
    204          {
    205          LinkedList<String> lines = new LinkedList<String>();
    206
    207          if( !localFile.isDirectory() )
    208            return populate( localFile, lines );
    209
    210          Collection<File> subFiles = FileUtils.listFiles( localFile, new RegexFileFilter( "^part-.*" ), null );
    211
    212          for( File subFile : subFiles )
    213            populate( subFile, lines );
    214
    215          return lines;
    216          }
    217
    218        private LinkedList<String> populate( File localFile, LinkedList<String> lines ) throws IOException
    219          {
    220          LineIterator iterator = FileUtils.lineIterator( localFile, "UTF-8" );
    221
    222          while( iterator.hasNext() )
    223            lines.add( iterator.next() );
    224
    225          return lines;
    226          }
    227        }
    228      }