    001    /*
    002     * Copyright (c) 2007-2014 Concurrent, Inc. All Rights Reserved.
    003     *
    004     * Project and contact information: http://www.cascading.org/
    005     *
    006     * This file is part of the Cascading project.
    007     *
    008     * Licensed under the Apache License, Version 2.0 (the "License");
    009     * you may not use this file except in compliance with the License.
    010     * You may obtain a copy of the License at
    011     *
    012     *     http://www.apache.org/licenses/LICENSE-2.0
    013     *
    014     * Unless required by applicable law or agreed to in writing, software
    015     * distributed under the License is distributed on an "AS IS" BASIS,
    016     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    017     * See the License for the specific language governing permissions and
    018     * limitations under the License.
    019     */
    020
    021    package cascading.stats.hadoop;
    022
    023    import java.util.HashMap;
    024    import java.util.Map;
    025
    026    import cascading.stats.CascadingStats;
    027    import org.apache.hadoop.mapred.Counters;
    028    import org.apache.hadoop.mapred.TaskCompletionEvent;
    029    import org.apache.hadoop.mapred.TaskReport;
    030
    031    import static cascading.stats.CascadingStats.Status.*;
    032
    033    /** Class HadoopTaskStats tracks individual task stats. */
    034    public class HadoopSliceStats
    035      {
    036      private final CascadingStats.Status parentStatus;
    037
    038      public static class HadoopAttempt
    039        {
    040        private final TaskCompletionEvent event;
    041
    042        public HadoopAttempt( TaskCompletionEvent event )
    043          {
    044          this.event = event;
    045          }
    046
    047        public int getEventId()
    048          {
    049          return event.getEventId();
    050          }
    051
    052        public int getTaskRunTime()
    053          {
    054          return event.getTaskRunTime();
    055          }
    056
    057        public String getTaskStatus()
    058          {
    059          return event.getTaskStatus().toString();
    060          }
    061
    062        public String getTaskTrackerHttp()
    063          {
    064          return event.getTaskTrackerHttp();
    065          }
    066
    067        public CascadingStats.Status getStatusFor()
    068          {
    069          CascadingStats.Status status = null;
    070
    071          switch( event.getTaskStatus() )
    072            {
    073            case FAILED:
    074              status = FAILED;
    075              break;
    076            case KILLED:
    077              status = STOPPED;
    078              break;
    079            case SUCCEEDED:
    080              status = SUCCESSFUL;
    081              break;
    082            case OBSOLETE:
    083              status = SKIPPED;
    084              break;
    085            case TIPFAILED:
    086              status = FAILED;
    087              break;
    088            }
    089          return status;
    090          }
    091        }
    092
    093      public enum Kind
    094        {
    095          SETUP, MAPPER, REDUCER, CLEANUP
    096        }
    097
    098      private String id;
    099      private Kind kind;
    100      private final boolean parentStepHasReducers;
    101      private TaskReport taskReport;
    102      private Map<String, Map<String, Long>> counters;
    103
    104      private Map<Integer, HadoopAttempt> attempts = new HashMap<Integer, HadoopAttempt>();
    105
    106      HadoopSliceStats( String id, CascadingStats.Status parentStatus, Kind kind, boolean parentStepHasReducers, TaskReport taskReport )
    107        {
    108        this.parentStatus = parentStatus;
    109        this.id = id;
    110        this.kind = kind;
    111        this.parentStepHasReducers = parentStepHasReducers;
    112        this.taskReport = taskReport;
    113        }
    114
    115      public String getID()
    116        {
    117        return id;
    118        }
    119
    120      public Kind getKind()
    121        {
    122        return kind;
    123        }
    124
    125      /**
    126       * Method getId returns the Hadoop task id.
    127       *
    128       * @return the id (type String) of this HadoopTaskStats object.
    129       */
    130      public String getTaskID()
    131        {
    132        return taskReport.getTaskID().toString();
    133        }
    134
    135      public int getTaskIDNum()
    136        {
    137        return taskReport.getTaskID().getId();
    138        }
    139
    140      public String getJobID()
    141        {
    142        return taskReport.getTaskID().getJobID().toString();
    143        }
    144
    145      public boolean parentStepHasReducers()
    146        {
    147        return parentStepHasReducers;
    148        }
    149
    150      public float getProgress()
    151        {
    152        return taskReport.getProgress();
    153        }
    154
    155      public String getState()
    156        {
    157        return taskReport.getState();
    158        }
    159
    160      public long getStartTime()
    161        {
    162        return taskReport.getStartTime();
    163        }
    164
    165      public long getFinishTime()
    166        {
    167        return taskReport.getFinishTime();
    168        }
    169
    170      public CascadingStats.Status getParentStatus()
    171        {
    172        return parentStatus;
    173        }
    174
    175      public CascadingStats.Status getStatus()
    176        {
    177        CascadingStats.Status status = null;
    178
    179        switch( taskReport.getCurrentStatus() )
    180          {
    181          case PENDING:
    182            status = PENDING;
    183            break;
    184          case RUNNING:
    185            status = RUNNING;
    186            break;
    187          case COMPLETE:
    188            status = SUCCESSFUL;
    189            break;
    190          case KILLED:
    191            status = STOPPED;
    192            break;
    193          case FAILED:
    194            status = FAILED;
    195            break;
    196          }
    197
    198        return status;
    199        }
    200
    201      public String[] getDiagnostics()
    202        {
    203        return taskReport.getDiagnostics();
    204        }
    205
    206      public Map<String, Map<String, Long>> getCounters()
    207        {
    208        if( counters == null )
    209          setCounters( taskReport );
    210
    211        return counters;
    212        }
    213
    214      public Map<Integer, HadoopAttempt> getAttempts()
    215        {
    216        return attempts;
    217        }
    218
    219      private void setCounters( TaskReport taskReport )
    220        {
    221        this.counters = new HashMap<String, Map<String, Long>>();
    222
    223        Counters hadoopCounters = taskReport.getCounters();
    224
    225        for( Counters.Group group : hadoopCounters )
    226          {
    227          Map<String, Long> values = new HashMap<String, Long>();
    228
    229          this.counters.put( group.getName(), values );
    230
    231          for( Counters.Counter counter : group )
    232            values.put( counter.getName(), counter.getCounter() );
    233          }
    234        }
    235
    236      /**
    237       * Method getCounterValue returns the raw Hadoop counter value.
    238       *
    239       * @param counter of Enum
    240       * @return long
    241       */
    242      public long getCounterValue( Enum counter )
    243        {
    244        return getCounterValue( counter.getDeclaringClass().getName(), counter.name() );
    245        }
    246
    247      /**
    248       * Method getCounterValue returns the raw Hadoop counter value.
    249       *
    250       * @param group of String
    251       * @param name  of String
    252       * @return long
    253       */
    254      public long getCounterValue( String group, String name )
    255        {
    256        if( getCounters() == null || getCounters().get( group ) == null )
    257          return 0;
    258
    259        Long value = getCounters().get( group ).get( name );
    260
    261        if( value == null )
    262          return 0;
    263
    264        return value;
    265        }
    266
    267      public void addAttempt( TaskCompletionEvent event )
    268        {
    269        attempts.put( event.getEventId(), new HadoopAttempt( event ) );
    270        }
    271
    272      @Override
    273      public String toString()
    274        {
    275        final StringBuilder sb = new StringBuilder();
    276        sb.append( "HadoopTaskStats" );
    277        sb.append( "{id='" ).append( id ).append( '\'' );
    278        sb.append( ", kind=" ).append( kind );
    279        sb.append( '}' );
    280        return sb.toString();
    281        }
    282      }