    001/*
    002 * Copyright (c) 2007-2016 Concurrent, Inc. All Rights Reserved.
    003 *
    004 * Project and contact information: http://www.cascading.org/
    005 *
    006 * This file is part of the Cascading project.
    007 *
    008 * Licensed under the Apache License, Version 2.0 (the "License");
    009 * you may not use this file except in compliance with the License.
    010 * You may obtain a copy of the License at
    011 *
    012 *     http://www.apache.org/licenses/LICENSE-2.0
    013 *
    014 * Unless required by applicable law or agreed to in writing, software
    015 * distributed under the License is distributed on an "AS IS" BASIS,
    016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    017 * See the License for the specific language governing permissions and
    018 * limitations under the License.
    019 */
    020
    021package cascading.flow.stream.element;
    022
    023import java.util.Collection;
    024import java.util.Collections;
    025import java.util.HashSet;
    026import java.util.Iterator;
    027import java.util.Set;
    028
    029import cascading.flow.FlowProcess;
    030import cascading.flow.stream.duct.Duct;
    031import cascading.pipe.Splice;
    032import cascading.tuple.Tuple;
    033import cascading.tuple.TupleEntry;
    034import cascading.tuple.Tuples;
    035
    036/**
    037 *
    038 */
    039public class MemoryCoGroupGate extends MemorySpliceGate
    040  {
    041  public MemoryCoGroupGate( FlowProcess flowProcess, Splice splice )
    042    {
    043    super( flowProcess, splice );
    044    }
    045
    046  @Override
    047  protected boolean isBlockingStreamed()
    048    {
    049    return true;
    050    }
    051
    052  @Override
    053  public void start( Duct previous )
    054    {
    055    }
    056
    057  @Override
    058  public void receive( Duct previous, TupleEntry incomingEntry )
    059    {
    060    int pos = ordinalMap.get( previous );
    061
    062    Tuple valuesTuple = incomingEntry.getTupleCopy();
    063    Tuple groupTuple = keyBuilder[ pos ].makeResult( valuesTuple, null ); // view on valuesTuple
    064
    065    groupTuple = getDelegatedTuple( groupTuple ); // wrap so hasher/comparator is honored
    066
    067    keys.add( groupTuple );
    068    keyValues[ pos ].get( groupTuple ).add( valuesTuple );
    069    }
    070
    071  @Override
    072  public void complete( Duct previous )
    073    {
    074    if( count.decrementAndGet() != 0 )
    075      return;
    076
    077    next.start( this );
    078
    079    Collection<Tuple>[] collections = new Collection[ keyValues.length ];
    080    Iterator<Tuple> keyIterator = keys.iterator();
    081
    082    Set<Tuple> seenNulls = new HashSet<Tuple>();
    083
    084    while( keyIterator.hasNext() )
    085      {
    086      Tuple keysTuple = keyIterator.next();
    087
    088      keyIterator.remove();
    089
    090      // provides sql like semantics
    091      if( nullsAreNotEqual && Tuples.frequency( keysTuple, null ) != 0 )
    092        {
    093        if( seenNulls.contains( keysTuple ) )
    094          continue;
    095
    096        seenNulls.add( keysTuple );
    097
    098        for( int i = 0; i < keyValues.length; i++ )
    099          {
    100          Collection<Tuple> values = keyValues[ i ].remove( keysTuple );
    101
    102          if( values == null )
    103            continue;
    104
    105          for( int j = 0; j < keyValues.length; j++ )
    106            collections[ j ] = Collections.emptyList();
    107
    108          collections[ i ] = values;
    109
    110          push( collections, keysTuple );
    111          }
    112        }
    113      else
    114        {
    115        // drain the keys and keyValues collections to preserve memory
    116        for( int i = 0; i < keyValues.length; i++ )
    117          {
    118          collections[ i ] = keyValues[ i ].remove( keysTuple );
    119
    120          if( collections[ i ] == null )
    121            collections[ i ] = Collections.emptyList();
    122          }
    123
    124        push( collections, keysTuple );
    125        }
    126      }
    127
    128    keys = createKeySet();
    129    keyValues = createKeyValuesArray();
    130
    131    count.set( numIncomingEventingPaths );
    132
    133    next.complete( this );
    134    }
    135
    136  private void push( Collection<Tuple>[] collections, Tuple keysTuple )
    137    {
    138    closure.reset( collections );
    139
    140    keyEntry.setTuple( closure.getGroupTuple( keysTuple ) );
    141
    142    // create Closure type here
    143    tupleEntryIterator.reset( splice.getJoiner().getIterator( closure ) );
    144
    145    next.receive( this, grouping );
    146    }
    147  }