    001    /*
    002     * Copyright (c) 2007-2014 Concurrent, Inc. All Rights Reserved.
    003     *
    004     * Project and contact information: http://www.cascading.org/
    005     *
    006     * This file is part of the Cascading project.
    007     *
    008     * Licensed under the Apache License, Version 2.0 (the "License");
    009     * you may not use this file except in compliance with the License.
    010     * You may obtain a copy of the License at
    011     *
    012     *     http://www.apache.org/licenses/LICENSE-2.0
    013     *
    014     * Unless required by applicable law or agreed to in writing, software
    015     * distributed under the License is distributed on an "AS IS" BASIS,
    016     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    017     * See the License for the specific language governing permissions and
    018     * limitations under the License.
    019     */
    020
    021    package cascading.flow.planner;
    022
    023    import java.io.IOException;
    024    import java.util.ArrayList;
    025    import java.util.List;
    026    import java.util.concurrent.Callable;
    027    import java.util.concurrent.CountDownLatch;
    028
    029    import cascading.flow.Flow;
    030    import cascading.flow.FlowException;
    031    import cascading.flow.FlowStep;
    032    import cascading.flow.FlowStepStrategy;
    033    import cascading.management.state.ClientState;
    034    import cascading.stats.FlowStats;
    035    import cascading.stats.FlowStepStats;
    036    import cascading.util.Util;
    037    import org.slf4j.Logger;
    038    import org.slf4j.LoggerFactory;
    039
    040    /**
    041     *
    042     */
    043    public abstract class FlowStepJob<Config> implements Callable<Throwable>
    044      {
    045      private static final Logger LOG = LoggerFactory.getLogger( FlowStepJob.class );
    046
    047      /** Field stepName */
    048      protected final String stepName;
    049      /** Field pollingInterval */
    050      protected long pollingInterval = 1000;
    051      /** Field recordStatsInterval */
    052      protected long statsStoreInterval = 60 * 1000;
    053      /** Field predecessors */
    054      protected List<FlowStepJob<Config>> predecessors;
    055      /** Field latch */
    056      private final CountDownLatch latch = new CountDownLatch( 1 );
    057      /** Field stop */
    058      private boolean stop = false;
    059      /** Field flowStep */
    060      protected final BaseFlowStep<Config> flowStep;
    061      /** Field stepStats */
    062      protected FlowStepStats flowStepStats;
    063      /** Field throwable */
    064      protected Throwable throwable;
    065
    066      public FlowStepJob( ClientState clientState, BaseFlowStep flowStep, long pollingInterval, long statsStoreInterval )
    067        {
    068        this.flowStep = flowStep;
    069        this.stepName = flowStep.getName();
    070        this.pollingInterval = pollingInterval;
    071        this.statsStoreInterval = statsStoreInterval;
    072        this.flowStepStats = createStepStats( clientState );
    073
    074        this.flowStepStats.prepare();
    075        this.flowStepStats.markPending();
    076        }
    077
    078      public abstract Config getConfig();
    079
    080      protected abstract FlowStepStats createStepStats( ClientState clientState );
    081
    082      public synchronized void stop()
    083        {
    084        if( flowStep.isInfoEnabled() )
    085          flowStep.logInfo( "stopping: " + stepName );
    086
    087        stop = true;
    088
    089        // allow pending -> stopped transition
    090        // never want a hanging pending state
    091        if( !flowStepStats.isFinished() )
    092          flowStepStats.markStopped();
    093
    094        try
    095          {
    096          internalBlockOnStop();
    097          }
    098        catch( IOException exception )
    099          {
    100          flowStep.logWarn( "unable to kill job: " + stepName, exception );
    101          }
    102        finally
    103          {
    104          // call rollback after the job has been stopped, only if it was stopped
    105          if( flowStepStats.isStopped() )
    106            {
    107            flowStep.rollbackSinks();
    108            flowStep.fireOnStopping();
    109            }
    110
    111          flowStepStats.cleanup();
    112          }
    113        }
    114
    115      protected abstract void internalBlockOnStop() throws IOException;
    116
    117      public void setPredecessors( List<FlowStepJob<Config>> predecessors )
    118        {
    119        this.predecessors = predecessors;
    120        }
    121
    122      public Throwable call()
    123        {
    124        start();
    125
    126        return throwable;
    127        }
    128
    129      protected void start()
    130        {
    131        try
    132          {
    133          if( isSkipFlowStep() )
    134            {
    135            markSkipped();
    136
    137            if( flowStep.isInfoEnabled() && flowStepStats.isSkipped() )
    138              flowStep.logInfo( "skipping step: " + stepName );
    139
    140            return;
    141            }
    142
    143          if( !markStarted() )
    144            return;
    145
    146          blockOnPredecessors();
    147
    148          applyFlowStepConfStrategy();
    149
    150          blockOnJob();
    151          }
    152        catch( Throwable throwable )
    153          {
    154          dumpDebugInfo();
    155          this.throwable = throwable;
    156          flowStep.fireOnThrowable( throwable );
    157          }
    158        finally
    159          {
    160          latch.countDown();
    161          flowStepStats.cleanup();
    162          }
    163        }
    164
    165      private synchronized boolean markStarted()
    166        {
    167        if( flowStepStats.isFinished() ) // if stopped, return
    168          return false;
    169
    170        flowStepStats.markStarted();
    171
    172        return true;
    173        }
    174
    175      private void applyFlowStepConfStrategy()
    176        {
    177        FlowStepStrategy flowStepStrategy = flowStep.getFlow().getFlowStepStrategy();
    178
    179        if( flowStepStrategy == null )
    180          return;
    181
    182        List<FlowStep> predecessorSteps = new ArrayList<FlowStep>();
    183
    184        for( FlowStepJob predecessor : predecessors )
    185          predecessorSteps.add( predecessor.flowStep );
    186
    187        flowStepStrategy.apply( flowStep.getFlow(), predecessorSteps, flowStep );
    188        }
    189
    190      protected boolean isSkipFlowStep() throws IOException
    191        {
    192        // if runID is not set, never skip a step
    193        if( flowStep.getFlow().getRunID() == null )
    194          return false;
    195
    196        return flowStep.allSourcesExist() && !flowStep.areSourcesNewer( flowStep.getSinkModified() );
    197        }
    198
    199      protected void blockOnJob() throws IOException
    200        {
    201        if( stop ) // true if a predecessor failed
    202          return;
    203
    204        if( flowStep.isInfoEnabled() )
    205          flowStep.logInfo( "starting step: " + stepName );
    206
    207        internalNonBlockingStart();
    208
    209        markSubmitted();
    210        flowStep.fireOnStarting();
    211
    212        blockTillCompleteOrStopped();
    213
    214        if( !stop && !internalNonBlockingIsSuccessful() )
    215          {
    216          if( !flowStepStats.isFinished() )
    217            {
    218            flowStep.rollbackSinks();
    219            flowStepStats.markFailed( getThrowable() );
    220            flowStep.fireOnThrowable( getThrowable() );
    221            }
    222
    223          dumpDebugInfo();
    224
    225          // if available, rethrow the unrecoverable error
    226          if( getThrowable() instanceof OutOfMemoryError )
    227            throw ( (OutOfMemoryError) getThrowable() );
    228
    229          if( !isRemoteExecution() )
    230            throwable = new FlowException( "local step failed", getThrowable() );
    231          else
    232            throwable = new FlowException( "step failed: " + stepName + ", with job id: " + internalJobId() + ", please see cluster logs for failure messages" );
    233          }
    234        else
    235          {
    236          if( internalNonBlockingIsSuccessful() && !flowStepStats.isFinished() )
    237            {
    238            throwable = flowStep.commitSinks();
    239
    240            if( throwable != null )
    241              {
    242              flowStepStats.markFailed( throwable );
    243              flowStep.fireOnThrowable( throwable );
    244              }
    245            else
    246              {
    247              flowStepStats.markSuccessful();
    248              flowStep.fireOnCompleted();
    249              }
    250            }
    251          }
    252
    253        flowStepStats.recordChildStats();
    254        }
    255
    256      protected abstract boolean isRemoteExecution();
    257
    258      protected abstract String internalJobId();
    259
    260      protected abstract boolean internalNonBlockingIsSuccessful() throws IOException;
    261
    262      protected abstract Throwable getThrowable();
    263
    264      protected abstract void internalNonBlockingStart() throws IOException;
    265
    266      protected void blockTillCompleteOrStopped() throws IOException
    267        {
    268        int iterations = (int) Math.floor( statsStoreInterval / pollingInterval );
    269        int count = 0;
    270
    271        while( true )
    272          {
    273          if( flowStepStats.isSubmitted() && isStarted() )
    274            {
    275            markRunning();
    276            flowStep.fireOnRunning();
    277            }
    278
    279          if( stop || internalNonBlockingIsComplete() )
    280            break;
    281
    282          sleepForPollingInterval();
    283
    284          if( iterations == count++ )
    285            {
    286            count = 0;
    287            flowStepStats.recordStats();
    288            flowStepStats.recordChildStats();
    289            }
    290          }
    291        }
    292
    293      private synchronized void markSubmitted()
    294        {
    295        if( flowStepStats.isStarted() )
    296          flowStepStats.markSubmitted();
    297
    298        Flow flow = flowStep.getFlow();
    299
    300        if( flow == null )
    301          {
    302          LOG.warn( "no parent flow set" );
    303          return;
    304          }
    305
    306        FlowStats flowStats = flow.getFlowStats();
    307
    308        synchronized( flowStats )
    309          {
    310          if( flowStats.isStarted() )
    311            flowStats.markSubmitted();
    312          }
    313        }
    314
    315      private synchronized void markRunning()
    316        {
    317        flowStepStats.markRunning();
    318
    319        markFlowRunning();
    320        }
    321
    322      private synchronized void markSkipped()
    323        {
    324        if( flowStepStats.isFinished() )
    325          return;
    326
    327        flowStepStats.markSkipped();
    328
    329        markFlowRunning();
    330        }
    331
    332      private synchronized void markFlowRunning()
    333        {
    334        Flow flow = flowStep.getFlow();
    335
    336        if( flow == null )
    337          {
    338          LOG.warn( "no parent flow set" );
    339          return;
    340          }
    341
    342        FlowStats flowStats = flow.getFlowStats();
    343
    344        synchronized( flowStats )
    345          {
    346          if( flowStats.isStarted() || flowStats.isSubmitted() )
    347            flowStats.markRunning();
    348          }
    349        }
    350
    351      protected abstract boolean internalNonBlockingIsComplete() throws IOException;
    352
    353      protected void sleepForPollingInterval()
    354        {
    355        Util.safeSleep( pollingInterval );
    356        }
    357
    358      protected void blockOnPredecessors()
    359        {
    360        for( FlowStepJob predecessor : predecessors )
    361          {
    362          if( !predecessor.isSuccessful() )
    363            {
    364            flowStep.logWarn( "abandoning step: " + stepName + ", predecessor failed: " + predecessor.stepName );
    365
    366            stop();
    367            }
    368          }
    369        }
    370
    371      protected abstract void dumpDebugInfo();
    372
    373      /**
    374       * Method isSuccessful returns true if this step completed successfully or was skipped.
    375       *
    376       * @return the successful (type boolean) of this FlowStepJob object.
    377       */
    378      public boolean isSuccessful()
    379        {
    380        try
    381          {
    382          latch.await(); // freed after step completes in #start()
    383
    384          return flowStepStats.isSuccessful() || flowStepStats.isSkipped();
    385          }
    386        catch( InterruptedException exception )
    387          {
    388          flowStep.logWarn( "latch interrupted", exception );
    389
    390          return false;
    391          }
    392        catch( NullPointerException exception )
    393          {
    394          throw new FlowException( "Hadoop is not keeping a large enough job history, please increase the \'mapred.jobtracker.completeuserjobs.maximum\' property", exception );
    395          }
    396        }
    397
    398      /**
    399       * Method wasStarted returns true if this job was started
    400       *
    401       * @return boolean
    402       */
    403      public boolean isStarted()
    404        {
    405        return internalIsStarted();
    406        }
    407
    408      protected abstract boolean internalIsStarted();
    409
    410      /**
    411       * Method getStepStats returns the stepStats of this FlowStepJob object.
    412       *
    413       * @return the stepStats (type StepStats) of this FlowStepJob object.
    414       */
    415      public FlowStepStats getStepStats()
    416        {
    417        return flowStepStats;
    418        }
    419      }