    001    /*
    002     * Copyright (c) 2007-2014 Concurrent, Inc. All Rights Reserved.
    003     *
    004     * Project and contact information: http://www.cascading.org/
    005     *
    006     * This file is part of the Cascading project.
    007     *
    008     * Licensed under the Apache License, Version 2.0 (the "License");
    009     * you may not use this file except in compliance with the License.
    010     * You may obtain a copy of the License at
    011     *
    012     *     http://www.apache.org/licenses/LICENSE-2.0
    013     *
    014     * Unless required by applicable law or agreed to in writing, software
    015     * distributed under the License is distributed on an "AS IS" BASIS,
    016     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    017     * See the License for the specific language governing permissions and
    018     * limitations under the License.
    019     */
    020
    021    package cascading.flow.local.stream;
    022
    023    import java.util.Collections;
    024    import java.util.Iterator;
    025    import java.util.List;
    026
    027    import cascading.flow.FlowProcess;
    028    import cascading.flow.stream.Duct;
    029    import cascading.flow.stream.MemorySpliceGate;
    030    import cascading.pipe.Splice;
    031    import cascading.tuple.Tuple;
    032    import cascading.tuple.TupleEntry;
    033    import com.google.common.collect.ArrayListMultimap;
    034    import com.google.common.collect.ListMultimap;
    035    import com.google.common.collect.Multimaps;
    036
    037    /**
    038     *
    039     */
    040    public class LocalGroupByGate extends MemorySpliceGate
    041      {
    042      private ListMultimap<Tuple, Tuple> valueMap;
    043
    044      public LocalGroupByGate( FlowProcess flowProcess, Splice splice )
    045        {
    046        super( flowProcess, splice );
    047        }
    048
    049      @Override
    050      protected boolean isBlockingStreamed()
    051        {
    052        return true;
    053        }
    054
    055      private ListMultimap<Tuple, Tuple> initNewValueMap()
    056        {
    057        return Multimaps.synchronizedListMultimap( ArrayListMultimap.<Tuple, Tuple>create() );
    058        }
    059
    060      @Override
    061      public void prepare()
    062        {
    063        super.prepare();
    064
    065        valueMap = initNewValueMap();
    066        }
    067
    068      @Override
    069      public void start( Duct previous )
    070        {
    071        // chained below in #complete()
    072        }
    073
    074      @Override
    075      public void receive( Duct previous, TupleEntry incomingEntry )
    076        {
    077        Tuple valuesTuple = incomingEntry.getTupleCopy();
    078        Tuple groupTuple = keyBuilder[ 0 ].makeResult( valuesTuple, null ); // view on valuesTuple
    079
    080        groupTuple = getDelegatedTuple( groupTuple ); // wrap so hasher/comparator is honored
    081
    082        keys.add( groupTuple );
    083        valueMap.put( groupTuple, valuesTuple );
    084        }
    085
    086      @Override
    087      public void complete( Duct previous )
    088        {
    089        if( count.decrementAndGet() != 0 )
    090          return;
    091
    092        next.start( this );
    093
    094        // drain the keys and keyValues collections to preserve memory
    095        Iterator<Tuple> iterator = keys.iterator();
    096
    097        // no need to synchronize here as we are guaranteed all writer threads are completed
    098        while( iterator.hasNext() )
    099          {
    100          Tuple groupTuple = iterator.next();
    101
    102          iterator.remove();
    103
    104          keyEntry.setTuple( groupTuple );
    105
    106          List<Tuple> tuples = valueMap.get( groupTuple ); // can't removeAll, returns unmodifiable collection
    107
    108          if( valueComparators != null )
    109            Collections.sort( tuples, valueComparators[ 0 ] );
    110
    111          tupleEntryIterator.reset( tuples.iterator() );
    112
    113          next.receive( this, grouping );
    114
    115          tuples.clear();
    116          }
    117
    118        keys = createKeySet();
    119        valueMap = initNewValueMap();
    120        count.set( numIncomingPaths );
    121
    122        next.complete( this );
    123        }
    124      }