Question

我有以下简单的flink应用程序在IDE中运行，我每5秒执行一个检查点，并想将检查点数据写入目录file:///d:/applog/out/mycheckpoint/，但是在运行了一段时间后，我停止了该应用程序，但在目录file:///d:/applog/out/mycheckpoint/

下未找到任何内容

代码是：

import java.util.Date

import io.github.streamingwithflink.util.DateUtil
import org.apache.flink.api.common.state.{ListState, ListStateDescriptor}
import org.apache.flink.api.scala._
import org.apache.flink.runtime.state.filesystem.FsStateBackend
import org.apache.flink.runtime.state.{FunctionInitializationContext, FunctionSnapshotContext}
import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction
import org.apache.flink.streaming.api.environment.CheckpointConfig.ExternalizedCheckpointCleanup
import org.apache.flink.streaming.api.functions.source.SourceFunction
import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
    object SourceFunctionExample {

      def main(args: Array[String]): Unit = {

        val env = StreamExecutionEnvironment.getExecutionEnvironment
        env.setParallelism(4)
        env.getCheckpointConfig.setCheckpointInterval(5 * 1000)
        env.getCheckpointConfig.enableExternalizedCheckpoints(ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION)
        env.setStateBackend(new FsStateBackend("file:///d:/applog/out/mycheckpoint/"))
        val numbers: DataStream[Long] = env.addSource(new ReplayableCountSource)
        numbers.print()

        env.execute()
      }

    }


class ReplayableCountSource extends SourceFunction[Long] with CheckpointedFunction {

  var isRunning: Boolean = true
  var cnt: Long = _
  var offsetState: ListState[Long] = _

  override def run(ctx: SourceFunction.SourceContext[Long]): Unit = {

    while (isRunning && cnt < Long.MaxValue) {
      ctx.getCheckpointLock.synchronized {
        // increment cnt
        cnt += 1
        ctx.collect(cnt)
      }
      Thread.sleep(200)
    }

  }

  override def cancel(): Unit = isRunning = false

  override def snapshotState(snapshotCtx: FunctionSnapshotContext): Unit = {
    println("snapshotState is called at " + DateUtil.format(new Date) + s", cnt is ${cnt}")
    // remove previous cnt
    offsetState.clear()
    // add current cnt
    offsetState.add(cnt)
  }

  override def initializeState(initCtx: FunctionInitializationContext): Unit = {
    // obtain operator list state to store the current cnt

    val desc = new ListStateDescriptor[Long]("offset", classOf[Long])
    offsetState = initCtx.getOperatorStateStore.getListState(desc)

    // initialize cnt variable from the checkpoint
    val it = offsetState.get()
    cnt = if (null == it || !it.iterator().hasNext) {
      -1L
    } else {
      it.iterator().next()
    }

    println("initializeState is called at " + DateUtil.format(new Date) + s", cnt is ${cnt}")
  }
}

Answer 1

我在Windows和Linux上测试了该应用程序，在这两种情况下，检查点文件均按预期方式创建。

请注意，如果检查点失败（例如由于某些权限错误或无效路径），程序将继续运行。

Flink记录一条WARN消息，但异常导致检查点失败。

在我的简单应用程序中没有创建检查点文件

1 个答案: