ClassNotFoundException：com.amazonaws.services.s3.AmazonS3从Flink应用程序与S3交互时

Question

我正在尝试使用Flink从S3下载一些Avro文件。但是我收到以下错误，不确定为什么。我已经将aws-java-sdk-core-1.11.342.jar和hadoop-aws-2.7.4.jar jar文件添加到flink lib目录中。没有s3交互时，该程序运行正常，但在尝试与s3交互时给出错误消息。这里可能是什么问题？

Flink群集版本：1.3.2

SBT依赖项：-

scalaVersion in ThisBuild := "2.11.11"

val flink_version = "1.3.2"
val slf4j_version = "1.7.25"
val playJson_version = "2.6.3"
val avro_version = "1.8.2"
val cassandra_version = "3.3.0"
val mongodb_version = "3.5.0"
val commons_version = "1.40"
val aws_sdk_version = "1.11.342"

val dependencies = Seq(
  "org.apache.flink" %% "flink-scala" % flink_version,
  "org.apache.flink" %% "flink-clients" % flink_version,
  "org.apache.flink" %% "flink-avro" % flink_version,
  "org.apache.hadoop" % "hadoop-aws" % "2.7.4"  % Provided,
  "org.apache.flink" %% "flink-hadoop-compatibility" % flink_version,
  "org.apache.flink" %% "flink-connector-cassandra" % flink_version,
  "org.mongodb" % "mongo-java-driver" % mongodb_version,
  "com.typesafe.play" %% "play-json" % playJson_version,
  "org.apache.avro" % "avro-mapred" % "1.7.1",
  "com.paulgoldbaum" %% "scala-influxdb-client" % "0.4.3",

  "com.amazonaws" % "aws-java-sdk-dynamodb" % aws_sdk_version,
  "com.amazonaws" % "aws-java-sdk-core" % aws_sdk_version % Provided
) ++ Seq(
  "org.specs2" %% "specs2-core" % "3.9.4" % Test,
  "org.mockito" % "mockito-all" % "1.10.19" % Test,
  "com.amazonaws" % "DynamoDBLocal" % "[1.11,2.0)" % Test,
  "org.slf4j" % "slf4j-api" % slf4j_version,
  "org.slf4j" % "slf4j-log4j12" % slf4j_version
)

错误：-

at org.apache.flink.runtime.jobmanager.JobManager.aroundReceive(JobManager.scala:125)
    at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516)
    at akka.actor.ActorCell.invoke(ActorCell.scala:487)
    at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238)
    at akka.dispatch.Mailbox.run(Mailbox.scala:220)
    at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397)
    at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
    at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
    at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
    at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
Caused by: org.apache.flink.runtime.JobException: Creating the input splits caused an error: com/amazonaws/services/s3/AmazonS3
    at org.apache.flink.runtime.executiongraph.ExecutionJobVertex.<init>(ExecutionJobVertex.java:247)
    at org.apache.flink.runtime.executiongraph.ExecutionGraph.attachJobGraph(ExecutionGraph.java:780)
    at org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder.buildGraph(ExecutionGraphBuilder.java:166)
    at org.apache.flink.runtime.jobmanager.JobManager.org$apache$flink$runtime$jobmanager$JobManager$$submitJob(JobManager.scala:1315)
    ... 19 more
Caused by: java.lang.NoClassDefFoundError: com/amazonaws/services/s3/AmazonS3
    at java.lang.Class.forName0(Native Method)
    at java.lang.Class.forName(Class.java:348)
    at org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:1821)
    at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:1786)
    at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1880)
    at org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2298)
    at org.apache.flink.runtime.fs.hdfs.HadoopFileSystem.getHadoopWrapperClassNameForFileSystem(HadoopFileSystem.java:486)
    at org.apache.flink.core.fs.FileSystem.getHadoopWrapperClassNameForFileSystem(FileSystem.java:437)
    at org.apache.flink.core.fs.FileSystem.getUnguardedFileSystem(FileSystem.java:339)
    at org.apache.flink.core.fs.FileSystem.get(FileSystem.java:389)
    at org.apache.flink.core.fs.Path.getFileSystem(Path.java:293)
    at org.apache.flink.api.common.io.FileInputFormat.createInputSplits(FileInputFormat.java:472)
    at org.apache.flink.api.common.io.FileInputFormat.createInputSplits(FileInputFormat.java:62)
    at org.apache.flink.runtime.executiongraph.ExecutionJobVertex.<init>(ExecutionJobVertex.java:233)
    ... 22 more
Caused by: java.lang.ClassNotFoundException: com.amazonaws.services.s3.AmazonS3
    at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
    ... 36 more