    001/*
    002 * Copyright (c) 2007-2016 Concurrent, Inc. All Rights Reserved.
    003 *
    004 * Project and contact information: http://www.cascading.org/
    005 *
    006 * This file is part of the Cascading project.
    007 *
    008 * Licensed under the Apache License, Version 2.0 (the "License");
    009 * you may not use this file except in compliance with the License.
    010 * You may obtain a copy of the License at
    011 *
    012 *     http://www.apache.org/licenses/LICENSE-2.0
    013 *
    014 * Unless required by applicable law or agreed to in writing, software
    015 * distributed under the License is distributed on an "AS IS" BASIS,
    016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    017 * See the License for the specific language governing permissions and
    018 * limitations under the License.
    019 */
    020
    021package cascading.flow.planner;
    022
    023import java.io.Serializable;
    024import java.util.HashSet;
    025import java.util.LinkedList;
    026import java.util.Map;
    027import java.util.Set;
    028
    029import cascading.tuple.Fields;
    030
    031import static cascading.tuple.Fields.asDeclaration;
    032
    033/** Class Scope is an internal representation of the linkages between operations. */
    034public class Scope implements Serializable
    035  {
    036  /** Enum Kind */
    037  public enum Kind
    038    {
    039      TAP, EACH, EVERY, GROUPBY, COGROUP, HASHJOIN, MERGE
    040    }
    041
    042  /** Field name */
    043  private String name;
    044  private LinkedList<String> priorNames = new LinkedList<>();
    045
    046  /** Field ordinal */
    047  private Integer ordinal = 0; // do not copy
    048  /** Field isStreamed is not accumulated. currently only relevant to HashJoin * */
    049  private boolean isNonBlocking = true; // do not copy
    050
    051  // copied
    052
    053  /** Field kind */
    054  private Kind kind;
    055  /** Field incomingPassThroughFields */
    056  private Fields incomingPassThroughFields;
    057  /** Field remainderPassThroughFields */
    058  private Fields remainderPassThroughFields;
    059
    060  /** Field argumentSelector */
    061  private Fields operationArgumentFields;
    062  /** Field declaredFields */
    063  private Fields operationDeclaredFields; // fields declared by the operation
    064
    065  /** Field groupingSelectors */
    066  private Map<String, Fields> keySelectors;
    067  /** Field sortingSelectors */
    068  private Map<String, Fields> sortingSelectors;
    069
    070  /** Field outGroupingSelector */
    071  private Fields outGroupingSelector;
    072  /** Field outGroupingFields */
    073  private Fields outGroupingFields; // all key fields
    074
    075  /** Field outValuesSelector */
    076  private Fields outValuesSelector;
    077  /** Field outValuesFields */
    078  private Fields outValuesFields; // all value fields, includes keys
    079
    080  /** Default constructor. */
    081  public Scope()
    082    {
    083    }
    084
    085  /**
    086   * Copy constructor
    087   *
    088   * @param scope of type Scope
    089   */
    090  public Scope( Scope scope )
    091    {
    092    this.name = scope.getName();
    093    copyFields( scope );
    094    }
    095
    096  /**
    097   * Tap constructor
    098   *
    099   * @param outFields of type Fields
    100   */
    101  public Scope( Fields outFields )
    102    {
    103    this.kind = Kind.TAP;
    104
    105    if( outFields == null )
    106      throw new IllegalArgumentException( "fields may not be null" );
    107
    108    this.outGroupingFields = outFields;
    109    this.outValuesFields = outFields;
    110    }
    111
    112  /**
    113   * Constructor Scope creates a new Scope instance. Used by classes Each and Every.
    114   *
    115   * @param name                      of type String
    116   * @param kind                      of type Kind
    117   * @param incomingPassThroughFields //   * @param remainderPassThroughFields   of type Fields
    118   * @param operationArgumentFields   of type Fields
    119   * @param operationDeclaredFields   of type Fields
    120   * @param outGroupingFields         of type Fields
    121   * @param outValuesFields           of type Fields
    122   */
    123  public Scope( String name, Kind kind, Fields incomingPassThroughFields, Fields remainderPassThroughFields, Fields operationArgumentFields, Fields operationDeclaredFields, Fields outGroupingFields, Fields outValuesFields )
    124    {
    125    this.name = name;
    126    this.kind = kind;
    127    this.incomingPassThroughFields = incomingPassThroughFields;
    128    this.remainderPassThroughFields = remainderPassThroughFields;
    129    this.operationArgumentFields = operationArgumentFields;
    130    this.operationDeclaredFields = operationDeclaredFields;
    131
    132    if( outGroupingFields == null )
    133      throw new IllegalArgumentException( "grouping may not be null" );
    134
    135    if( outValuesFields == null )
    136      throw new IllegalArgumentException( "values may not be null" );
    137
    138    if( kind == Kind.EACH || kind == Kind.EVERY )
    139      {
    140      this.outGroupingSelector = outGroupingFields;
    141      this.outGroupingFields = asDeclaration( outGroupingFields );
    142      this.outValuesSelector = outValuesFields;
    143      this.outValuesFields = asDeclaration( outValuesFields );
    144      }
    145    else
    146      {
    147      throw new IllegalArgumentException( "may not use the constructor for kind: " + kind );
    148      }
    149    }
    150
    151  /**
    152   * Constructor Scope creates a new Scope instance. Used by the Group class.
    153   *
    154   * @param name                    of type String
    155   * @param operationDeclaredFields of type Fields
    156   * @param outGroupingFields       of type Fields
    157   * @param keySelectors            of type Map<String, Fields>
    158   * @param sortingSelectors        of type Fields
    159   * @param outValuesFields         of type Fields
    160   * @param kind                    of type boolean
    161   */
    162  public Scope( String name, Fields operationDeclaredFields, Fields outGroupingFields, Map<String, Fields> keySelectors, Map<String, Fields> sortingSelectors, Fields outValuesFields, Kind kind )
    163    {
    164    this.name = name;
    165    this.kind = kind;
    166
    167    if( keySelectors == null )
    168      throw new IllegalArgumentException( "grouping may not be null" );
    169
    170    if( outValuesFields == null )
    171      throw new IllegalArgumentException( "values may not be null" );
    172
    173    this.operationDeclaredFields = operationDeclaredFields;
    174    this.outGroupingFields = asDeclaration( outGroupingFields );
    175    this.keySelectors = keySelectors;
    176    this.sortingSelectors = sortingSelectors; // null ok
    177    this.outValuesFields = asDeclaration( outValuesFields );
    178    }
    179
    180  /**
    181   * Constructor Scope creates a new Scope instance.
    182   *
    183   * @param name of type String
    184   */
    185  public Scope( String name )
    186    {
    187    this.name = name;
    188    }
    189
    190  public void addPriorNames( Scope incoming, Scope outgoing )
    191    {
    192    priorNames.addAll( 0, outgoing.priorNames );
    193    priorNames.addFirst( incoming.getName() );
    194    priorNames.addAll( 0, incoming.priorNames );
    195    }
    196
    197  public String getPriorName()
    198    {
    199    if( priorNames.isEmpty() )
    200      return getName();
    201
    202    return priorNames.getFirst();
    203    }
    204
    205  public Integer getOrdinal()
    206    {
    207    return ordinal;
    208    }
    209
    210  public void setOrdinal( Integer ordinal )
    211    {
    212    this.ordinal = ordinal;
    213    }
    214
    215  public void setNonBlocking( boolean isNonBlocking )
    216    {
    217    this.isNonBlocking = isNonBlocking;
    218    }
    219
    220  public boolean isNonBlocking()
    221    {
    222    return isNonBlocking;
    223    }
    224
    225  public boolean isSplice()
    226    {
    227    return isGroupBy() || isCoGroup() || isMerge() || isHashJoin();
    228    }
    229
    230  public boolean isGroup()
    231    {
    232    return kind == Kind.GROUPBY || kind == Kind.COGROUP;
    233    }
    234
    235  public boolean isGroupBy()
    236    {
    237    return kind == Kind.GROUPBY;
    238    }
    239
    240  public boolean isCoGroup()
    241    {
    242    return kind == Kind.COGROUP;
    243    }
    244
    245  public boolean isMerge()
    246    {
    247    return kind == Kind.MERGE;
    248    }
    249
    250  public boolean isHashJoin()
    251    {
    252    return kind == Kind.HASHJOIN;
    253    }
    254
    255  /**
    256   * Method isEach returns true if this Scope object represents an Each.
    257   *
    258   * @return the each (type boolean) of this Scope object.
    259   */
    260  public boolean isEach()
    261    {
    262    return kind == Kind.EACH;
    263    }
    264
    265  /**
    266   * Method isEvery returns true if this Scope object represents an Every.
    267   *
    268   * @return the every (type boolean) of this Scope object.
    269   */
    270  public boolean isEvery()
    271    {
    272    return kind == Kind.EVERY;
    273    }
    274
    275  /**
    276   * Method isTap returns true if this Scope object represents a Tap.
    277   *
    278   * @return the tap (type boolean) of this Scope object.
    279   */
    280  public boolean isTap()
    281    {
    282    return kind == Kind.TAP;
    283    }
    284
    285  /**
    286   * Method getName returns the name of this Scope object.
    287   *
    288   * @return the name (type String) of this Scope object.
    289   */
    290  public String getName()
    291    {
    292    return name;
    293    }
    294
    295  /**
    296   * Method setName sets the name of this Scope object.
    297   *
    298   * @param name the name of this Scope object.
    299   */
    300  public void setName( String name )
    301    {
    302    this.name = name;
    303    }
    304
    305  /**
    306   * Method getRemainderFields returns the remainderFields of this Scope object.
    307   *
    308   * @return the remainderFields (type Fields) of this Scope object.
    309   */
    310  public Fields getRemainderPassThroughFields()
    311    {
    312    return remainderPassThroughFields;
    313    }
    314
    315  /**
    316   * Method getArgumentSelector returns the argumentSelector of this Scope object.
    317   *
    318   * @return the argumentSelector (type Fields) of this Scope object.
    319   */
    320  public Fields getArgumentsSelector()
    321    {
    322    return operationArgumentFields;
    323    }
    324
    325  /**
    326   * Method getArguments returns the arguments of this Scope object.
    327   *
    328   * @return the arguments (type Fields) of this Scope object.
    329   */
    330  public Fields getArgumentsDeclarator()
    331    {
    332    return asDeclaration( operationArgumentFields );
    333    }
    334
    335  /**
    336   * Method getDeclaredFields returns the declaredFields of this Scope object.
    337   *
    338   * @return the declaredFields (type Fields) of this Scope object.
    339   */
    340  public Fields getOperationDeclaredFields()
    341    {
    342    return operationDeclaredFields;
    343    }
    344
    345  /**
    346   * Method getGroupingSelectors returns the groupingSelectors of this Scope object.
    347   *
    348   * @return the groupingSelectors (type Map<String, Fields>) of this Scope object.
    349   */
    350  public Map<String, Fields> getKeySelectors()
    351    {
    352    return keySelectors;
    353    }
    354
    355  /**
    356   * Method getSortingSelectors returns the sortingSelectors of this Scope object.
    357   *
    358   * @return the sortingSelectors (type Map<String, Fields>) of this Scope object.
    359   */
    360  public Map<String, Fields> getSortingSelectors()
    361    {
    362    return sortingSelectors;
    363    }
    364
    365  /**
    366   * Method getOutGroupingSelector returns the outGroupingSelector of this Scope object.
    367   *
    368   * @return the outGroupingSelector (type Fields) of this Scope object.
    369   */
    370  public Fields getOutGroupingSelector()
    371    {
    372    return outGroupingSelector;
    373    }
    374
    375  public Fields getIncomingTapFields()
    376    {
    377    if( isEvery() )
    378      return getOutGroupingFields();
    379    else
    380      return getOutValuesFields();
    381    }
    382
    383  public Fields getIncomingFunctionArgumentFields()
    384    {
    385    if( isEvery() )
    386      return getOutGroupingFields();
    387    else
    388      return getOutValuesFields();
    389    }
    390
    391  public Fields getIncomingFunctionPassThroughFields()
    392    {
    393    if( isEvery() )
    394      return getOutGroupingFields();
    395    else
    396      return getOutValuesFields();
    397    }
    398
    399  public Fields getIncomingAggregatorArgumentFields()
    400    {
    401    if( isEach() || isTap() )
    402      throw new IllegalStateException( "Every cannot follow a Tap or an Each" );
    403
    404    return getOutValuesFields();
    405    }
    406
    407  public Fields getIncomingAggregatorPassThroughFields()
    408    {
    409    if( isEach() || isTap() )
    410      throw new IllegalStateException( "Every cannot follow a Tap or an Each" );
    411
    412    return getOutGroupingFields();
    413    }
    414
    415  public Fields getIncomingBufferArgumentFields()
    416    {
    417    if( isEach() || isTap() )
    418      throw new IllegalStateException( "Every cannot follow a Tap or an Each" );
    419
    420    return getOutValuesFields();
    421    }
    422
    423  public Fields getIncomingBufferPassThroughFields()
    424    {
    425    if( isEach() || isTap() )
    426      throw new IllegalStateException( "Every cannot follow a Tap or an Each" );
    427
    428    return getOutValuesFields();
    429    }
    430
    431  public Fields getIncomingSpliceFields()
    432    {
    433    if( isEvery() )
    434      return getOutGroupingFields();
    435    else
    436      return getOutValuesFields();
    437    }
    438
    439  /**
    440   * Method getOutGroupingFields returns the outGroupingFields of this Scope object.
    441   *
    442   * @return the outGroupingFields (type Fields) of this Scope object.
    443   */
    444  public Fields getOutGroupingFields()
    445    {
    446    if( !isSplice() )
    447      return outGroupingFields;
    448
    449    Fields first = keySelectors.values().iterator().next();
    450
    451    // if more than one, this is a merge, so same key names are expected
    452    if( keySelectors.size() == 1 || isGroupBy() )
    453      return first;
    454
    455    // handling CoGroup only
    456
    457    // if given by user as resultGroupFields
    458    if( outGroupingFields != null )
    459      return outGroupingFields;
    460
    461    // todo throw an exception if we make it this far
    462
    463    // if all have the same names, then use for grouping
    464    Set<Fields> set = new HashSet<Fields>( keySelectors.values() );
    465
    466    if( set.size() == 1 )
    467      return first;
    468
    469    return Fields.size( first.size() );
    470    }
    471
    472  public Fields getOutGroupingValueFields()
    473    {
    474    return getOutValuesFields().subtract( getOutGroupingFields() );
    475    }
    476
    477  /**
    478   * Method getOutValuesSelector returns the outValuesSelector of this Scope object.
    479   *
    480   * @return the outValuesSelector (type Fields) of this Scope object.
    481   */
    482  public Fields getOutValuesSelector()
    483    {
    484    return outValuesSelector;
    485    }
    486
    487  /**
    488   * Method getOutValuesFields returns the outValuesFields of this Scope object.
    489   *
    490   * @return the outValuesFields (type Fields) of this Scope object.
    491   */
    492  public Fields getOutValuesFields()
    493    {
    494    return outValuesFields;
    495    }
    496
    497  /**
    498   * Method copyFields copies the given Scope instance fields to this instance.
    499   *
    500   * @param scope of type Scope
    501   */
    502  public void copyFields( Scope scope )
    503    {
    504    this.kind = scope.kind;
    505    this.incomingPassThroughFields = scope.incomingPassThroughFields;
    506    this.remainderPassThroughFields = scope.remainderPassThroughFields;
    507    this.operationArgumentFields = scope.operationArgumentFields;
    508    this.operationDeclaredFields = scope.operationDeclaredFields;
    509    this.keySelectors = scope.keySelectors;
    510    this.sortingSelectors = scope.sortingSelectors;
    511    this.outGroupingSelector = scope.outGroupingSelector;
    512    this.outGroupingFields = scope.outGroupingFields;
    513    this.outValuesSelector = scope.outValuesSelector;
    514    this.outValuesFields = scope.outValuesFields;
    515    }
    516
    517  @Override
    518  public String toString()
    519    {
    520    return print();
    521    }
    522
    523  public String print()
    524    {
    525    StringBuilder buffer = new StringBuilder();
    526
    527    if( ordinal != null )
    528      buffer.append( "[" ).append( ordinal ).append( "]" );
    529
    530    if( getOutValuesFields() == null )
    531      return buffer.toString(); // endpipes
    532
    533    buffer.append( "\n" );
    534
    535    if( keySelectors != null && !keySelectors.isEmpty() )
    536      {
    537      for( String name : keySelectors.keySet() )
    538        {
    539        if( buffer.length() != 0 && buffer.charAt( buffer.length() - 1 ) != '\n' )
    540          buffer.append( " | " );
    541
    542        buffer.append( name ).append( keySelectors.get( name ).printVerbose() );
    543        }
    544
    545      buffer.append( "\n" );
    546      }
    547
    548    if( outGroupingFields != null )
    549      buffer.append( getOutGroupingFields().printVerbose() ).append( "\n" );
    550
    551    buffer.append( getOutValuesFields().printVerbose() );
    552
    553    return buffer.toString();
    554    }
    555  }