    001    /*
    002     * Copyright (c) 2007-2014 Concurrent, Inc. All Rights Reserved.
    003     *
    004     * Project and contact information: http://www.cascading.org/
    005     *
    006     * This file is part of the Cascading project.
    007     *
    008     * Licensed under the Apache License, Version 2.0 (the "License");
    009     * you may not use this file except in compliance with the License.
    010     * You may obtain a copy of the License at
    011     *
    012     *     http://www.apache.org/licenses/LICENSE-2.0
    013     *
    014     * Unless required by applicable law or agreed to in writing, software
    015     * distributed under the License is distributed on an "AS IS" BASIS,
    016     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    017     * See the License for the specific language governing permissions and
    018     * limitations under the License.
    019     */
    020
    021    package cascading.stats.hadoop;
    022
    023    import java.util.HashMap;
    024    import java.util.Map;
    025
    026    import cascading.stats.CascadingStats;
    027    import org.apache.hadoop.mapreduce.Counter;
    028    import org.apache.hadoop.mapreduce.CounterGroup;
    029    import org.apache.hadoop.mapreduce.Counters;
    030    import org.apache.hadoop.mapreduce.TaskCompletionEvent;
    031    import org.apache.hadoop.mapreduce.TaskReport;
    032
    033    import static cascading.stats.CascadingStats.Status.*;
    034
    035    /** Class HadoopTaskStats tracks individual task stats. */
    036    public class HadoopSliceStats
    037      {
    038      private final CascadingStats.Status parentStatus;
    039
    040      public static class HadoopAttempt
    041        {
    042        private final TaskCompletionEvent event;
    043
    044        public HadoopAttempt( TaskCompletionEvent event )
    045          {
    046          this.event = event;
    047          }
    048
    049        public int getEventId()
    050          {
    051          return event.getEventId();
    052          }
    053
    054        public int getTaskRunTime()
    055          {
    056          return event.getTaskRunTime();
    057          }
    058
    059        public String getTaskStatus()
    060          {
    061          return event.getStatus().toString();
    062          }
    063
    064        public String getTaskTrackerHttp()
    065          {
    066          return event.getTaskTrackerHttp();
    067          }
    068
    069        public CascadingStats.Status getStatusFor()
    070          {
    071          CascadingStats.Status status = null;
    072
    073          switch( event.getStatus() )
    074            {
    075            case FAILED:
    076              status = FAILED;
    077              break;
    078            case KILLED:
    079              status = STOPPED;
    080              break;
    081            case SUCCEEDED:
    082              status = SUCCESSFUL;
    083              break;
    084            case OBSOLETE:
    085              status = SKIPPED;
    086              break;
    087            case TIPFAILED:
    088              status = FAILED;
    089              break;
    090            }
    091          return status;
    092          }
    093        }
    094
    095      public enum Kind
    096        {
    097          SETUP, MAPPER, REDUCER, CLEANUP
    098        }
    099
    100      private String id;
    101      private Kind kind;
    102      private final boolean parentStepHasReducers;
    103      private TaskReport taskReport;
    104      private Map<String, Map<String, Long>> counters;
    105
    106      private Map<Integer, HadoopAttempt> attempts = new HashMap<Integer, HadoopAttempt>();
    107
    108      HadoopSliceStats( String id, CascadingStats.Status parentStatus, Kind kind, boolean parentStepHasReducers, TaskReport taskReport )
    109        {
    110        this.parentStatus = parentStatus;
    111        this.id = id;
    112        this.kind = kind;
    113        this.parentStepHasReducers = parentStepHasReducers;
    114        this.taskReport = taskReport;
    115        }
    116
    117      public String getID()
    118        {
    119        return id;
    120        }
    121
    122      public Kind getKind()
    123        {
    124        return kind;
    125        }
    126
    127      /**
    128       * Method getId returns the Hadoop task id.
    129       *
    130       * @return the id (type String) of this HadoopTaskStats object.
    131       */
    132      public String getTaskID()
    133        {
    134        return taskReport.getTaskID().toString();
    135        }
    136
    137      public int getTaskIDNum()
    138        {
    139        return taskReport.getTaskID().getId();
    140        }
    141
    142      public String getJobID()
    143        {
    144        return taskReport.getTaskID().getJobID().toString();
    145        }
    146
    147      public boolean parentStepHasReducers()
    148        {
    149        return parentStepHasReducers;
    150        }
    151
    152      public float getProgress()
    153        {
    154        return taskReport.getProgress();
    155        }
    156
    157      public String getState()
    158        {
    159        return taskReport.getState();
    160        }
    161
    162      public long getStartTime()
    163        {
    164        return taskReport.getStartTime();
    165        }
    166
    167      public long getFinishTime()
    168        {
    169        return taskReport.getFinishTime();
    170        }
    171
    172      public CascadingStats.Status getParentStatus()
    173        {
    174        return parentStatus;
    175        }
    176
    177      public CascadingStats.Status getStatus()
    178        {
    179        CascadingStats.Status status = null;
    180
    181        switch( taskReport.getCurrentStatus() )
    182          {
    183          case PENDING:
    184            status = PENDING;
    185            break;
    186          case RUNNING:
    187            status = RUNNING;
    188            break;
    189          case COMPLETE:
    190            status = SUCCESSFUL;
    191            break;
    192          case KILLED:
    193            status = STOPPED;
    194            break;
    195          case FAILED:
    196            status = FAILED;
    197            break;
    198          }
    199
    200        return status;
    201        }
    202
    203      public String[] getDiagnostics()
    204        {
    205        return taskReport.getDiagnostics();
    206        }
    207
    208      public Map<String, Map<String, Long>> getCounters()
    209        {
    210        if( counters == null )
    211          setCounters( taskReport );
    212
    213        return counters;
    214        }
    215
    216      public Map<Integer, HadoopAttempt> getAttempts()
    217        {
    218        return attempts;
    219        }
    220
    221      private void setCounters( TaskReport taskReport )
    222        {
    223        this.counters = new HashMap<String, Map<String, Long>>();
    224
    225        Counters hadoopCounters = taskReport.getTaskCounters();
    226
    227        for( CounterGroup group : hadoopCounters )
    228          {
    229          Map<String, Long> values = new HashMap<String, Long>();
    230          this.counters.put( group.getName(), values );
    231
    232          for( Counter counter : group )
    233            values.put( counter.getName(), counter.getValue() );
    234          }
    235        }
    236
    237      /**
    238       * Method getCounterValue returns the raw Hadoop counter value.
    239       *
    240       * @param counter of Enum
    241       * @return long
    242       */
    243      public long getCounterValue( Enum counter )
    244        {
    245        return getCounterValue( counter.getDeclaringClass().getName(), counter.name() );
    246        }
    247
    248      /**
    249       * Method getCounterValue returns the raw Hadoop counter value.
    250       *
    251       * @param group of String
    252       * @param name  of String
    253       * @return long
    254       */
    255      public long getCounterValue( String group, String name )
    256        {
    257        if( getCounters() == null || getCounters().get( group ) == null )
    258          return 0;
    259
    260        Long value = getCounters().get( group ).get( name );
    261
    262        if( value == null )
    263          return 0;
    264
    265        return value;
    266        }
    267
    268      public void addAttempt( TaskCompletionEvent event )
    269        {
    270        attempts.put( event.getEventId(), new HadoopAttempt( event ) );
    271        }
    272
    273      @Override
    274      public String toString()
    275        {
    276        final StringBuilder sb = new StringBuilder();
    277        sb.append( "HadoopTaskStats" );
    278        sb.append( "{id='" ).append( id ).append( '\'' );
    279        sb.append( ", kind=" ).append( kind );
    280        sb.append( '}' );
    281        return sb.toString();
    282        }
    283      }