如何使用log4j自定义附加程序在HDFS上创建日志？

Question

概述

我们想使用log4j记录我们的Spark作业活动，以将日志文件写入HDFS。

• Java 8，Spark 2.4.6，Scala 2.1.2，Hadoop 3.2.1

我们无法找到要写入HDFS的本地apache log4j附加程序（不提供Apache Flume），因此请自行编写。

Scala逻辑与Java匹配，但是，登录到自定义附加程序始终失败，并出现堆栈溢出错误。

我们找不到明显不对的地方吗？

实施：

这里是一个简单的hello-world项目，用于重现该问题：

• Scala / Java项目（Maven构建），其中包含一个简单的主类，并且：
• HDFSAppender类（扩展了log4j AppenderSkeleton）
• LoggingTest单元测试类
• 用于测试的log4j配置文件

HDFSAppender.scala-Appender类

```
package com.obfuscated.test.spark.log4j

import java.io.BufferedOutputStream
import java.io.IOException
import java.net.URI
import java.net.URISyntaxException

import scala.beans.BeanProperty

import org.apache.hadoop.fs.FSDataOutputStream
import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.Path
import org.apache.hadoop.conf.Configuration
import org.apache.log4j.AppenderSkeleton
import org.apache.log4j.Level
import org.apache.log4j.Logger
import org.apache.log4j.MDC
import org.apache.log4j.spi.LoggingEvent
import org.apache.logging.log4j.scala.Logging
import org.apache.spark.SparkContext
import org.apache.spark.sql.SparkSession

object HDFSAppender {
  def createAndRegister(): HDFSAppender = {
    val hdfsAppender: HDFSAppender = new HDFSAppender
    Logger.getRootLogger.addAppender(hdfsAppender)
    hdfsAppender
  }

  def setClassLoggerLevel(clazz: Class[_], logLevel: Level): Level = {
    val logger = Logger.getLogger(clazz)
    val previousLevel = logger.getLevel
    Logger.getLogger(clazz).setLevel(logLevel)
    previousLevel
  }

  def unregister(hdfsAppender: HDFSAppender): Unit = {
    Logger.getRootLogger.removeAppender(hdfsAppender)
  }
}


class HDFSAppender extends AppenderSkeleton with Logging {

  var sc: SparkContext = _
  
  @BeanProperty
  var file: String = null

  @BeanProperty
  var URI: String = null

  override def close(): Unit = {}

  override def requiresLayout(): Boolean = true

  override protected def append(event: LoggingEvent): Unit = {
    var success: Boolean = false
    val log: String = this.layout.format(event)
    val uri: URI = new URI(this.URI)
    
    write(this.URI, file, log)
    
  }
    
  def write(uriString: String, file: String, log: String): Unit = {
    val conf: Configuration = new Configuration()
    MDC.put("eventName", "HDFSLogWriterWrite");
    
    
    val uri: URI = new URI(uriString)
    val logPath: Path = new Path(uriString + file)
    
    val fs = getFS(uri,conf)
    var output_stream: FSDataOutputStream = null
    try {
      if ((fs.exists(logPath) && fs.getFileStatus(logPath).isFile)) {
        output_stream = fs.append(logPath, 512)
      } else {
        output_stream = fs.create(logPath, true, 512)
      }
      val buffered_output: BufferedOutputStream = new BufferedOutputStream(output_stream)
      buffered_output.write(log.getBytes("UTF-8"))
      buffered_output.close()   
      
    } catch {
      
      case e: IOException => {
            e.printStackTrace();
      }
       case e: URISyntaxException => {
            e.printStackTrace();
      }
    }
  }
    /**
   * return a FileSystem from URI
   */
  private def getFS(uri: URI, conf: Configuration): FileSystem = {
    var fs: FileSystem = null
    try {
      fs = FileSystem.get(uri, conf)
    } catch {
      case e: Exception =>
        e.printStackTrace()
    }
    fs
  }
}

测试课程

```
package com.obfuscated.test.spark.log4j.test

import java.net.URI

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.Path
import org.apache.log4j.Level
import org.apache.log4j.LogManager
import org.apache.log4j.Logger
import org.junit.Assert.assertTrue
import org.junit.BeforeClass
import org.junit.Test
import org.scalatest.junit.JUnitSuite

import com.obfuscated.test.spark.log4j.CustomJsonEventLayout
import com.obfuscated.test.spark.log4j.HDFSAppender

import LoggingTest.fsHDFS
import LoggingTest.testHDFSHost
import LoggingTest.testOutputLogPath


object LoggingTest {
  
  var fsHDFS: FileSystem = _
  
  val testHDFSHost: String = 
    "hdfs://localhost:9000"
  
  val testOutputLogPath: String =
    "/test/ut/logging/log-test-hdfs-appender_scala.txt"

  @BeforeClass
  def setupFsRemovePreviousTestFiles(): Unit = {
    
    val conf: Configuration = new Configuration()
    conf.set("fs.defaultFS", testHDFSHost)
    fsHDFS = FileSystem.get(new URI(testHDFSHost), conf)
    val outputPath: Path = new Path(testOutputLogPath)
    if (fsHDFS.exists(outputPath)) {
      fsHDFS.delete(outputPath, true)
    }
  }

}

/**
 * programmatically builds and tests logging events with log4j.test
 *
 */
class LoggingTest extends JUnitSuite {
  val conf: Configuration = new Configuration()

  @Test
  def testHDFSAppender(): Unit = {
    val hdfsAppender: HDFSAppender = new HDFSAppender()
    val rootlogger: Logger = LogManager.getRootLogger
   
    hdfsAppender.setFile(testOutputLogPath)
    hdfsAppender.setURI(testHDFSHost)
    hdfsAppender.setThreshold(Level.ALL)
    hdfsAppender.setLayout(new CustomJsonEventLayout())
    rootlogger.addAppender(hdfsAppender)
    val logger: Logger = Logger.getLogger(this.getClass)

    try {
      logger.info("Test info message")
      logger.warn("Test warn message")
      logger.error("Test error message")
      logger.debug("Test debug message")
    } finally {
     logger.removeAppender(hdfsAppender)
  }
    val logFile: Path = new Path(testHDFSHost + testOutputLogPath)
    assertTrue(fsHDFS.exists(logFile))
    // TODO: further assertions to check log contents

  }
}

log4j配置文件（运行时测试）

```
log4j.rootLogger=DEBUG, console, HDFSAppender

log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.threshold=DEBUG
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5p] [%X{eventName}] %m %c%n    
log4j.appender.HDFSAppender=com.obfuscated.test.spark.log4j.HDFSAppender
log4j.appender.HDFSAppender.layout=com.obfuscated.test.spark.log4j.CustomJsonEventLayout
log4j.appender.HDFSAppender.file=/test/ut/logging/log-test-hdfs-appender_scala.json
log4j.appender.HDFSAppender.URI=hdfs://localhost:9000
log4j.appender.HDFSAppender.threshold=DEBUG

任何日志记录事件都会导致程序崩溃并出现堆栈溢出错误：

java.util.concurrent.ExecutionException: java.lang.StackOverflowError
 at java.util.concurrent.FutureTask.report(FutureTask.java:122)
 at java.util.concurrent.FutureTask.get(FutureTask.java:206)
 at org.apache.hadoop.util.ShutdownHookManager.executeShutdown(ShutdownHookManager.java:124)
 at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:95)
Caused by: java.lang.StackOverflowError
 at org.apache.log4j.PatternLayout.format(PatternLayout.java:506)
 at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:310)
 at org.apache.log4j.WriterAppender.append(WriterAppender.java:162)

然后出现以下消息，暗示某种循环/竞赛条件

 at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)
   at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
   at org.apache.log4j.Category.callAppenders(Category.java:206)
   at org.apache.log4j.Category.forcedLog(Category.java:391)
   at org.apache.log4j.Category.log(Category.java:856)
   at org.slf4j.impl.Log4jLoggerAdapter.warn(Log4jLoggerAdapter.java:401)
   at org.apache.spark.internal.Logging.logWarning(Logging.scala:66)
   at org.apache.spark.internal.Logging.logWarning$(Logging.scala:65)
   at org.apache.spark.SparkContext$.logWarning(SparkContext.scala:2442)
   at org.apache.spark.SparkContext$.$anonfun$assertNoOtherContextIsRunning$5(SparkContext.scala:2500)
   at org.apache.spark.SparkContext$.$anonfun$assertNoOtherContextIsRunning$5$adapted(SparkContext.scala:2491)
   at scala.Option.foreach(Option.scala:274)
   at org.apache.spark.SparkContext$.assertNoOtherContextIsRunning(SparkContext.scala:2491)
   at org.apache.spark.SparkContext$.markPartiallyConstructed(SparkContext.scala:2568)
   at org.apache.spark.SparkContext.<init>(SparkContext.scala:85)
   at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2520)
   at org.apache.spark.sql.SparkSession$Builder.$anonfun$getOrCreate$5(SparkSession.scala:935)
   at scala.Option.getOrElse(Option.scala:138)
   at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:926)
   at com.obfuscated.test.spark.log4j.HDFSLogWriter.write(HDFSLogWriter.scala:23)
   at com.obfuscated.test.spark.log4j.HDFSAppender.append(HDFSAppender.scala:63)
(repeats 48 times...)

Maven依赖项

Downloaded from central: 

https://repo.maven.apache.org/maven2/net/alchim31/maven/scala-maven-plugin/3.2.2/scala-maven-plugin-3.2.2.jar (119 kB at 163 kB/s)
[INFO] 
[INFO] ----------< spark-hello-world-scala:spark-hello-world-scala >-----------
[INFO] Building spark-hello-world-scala 0.0.1-SNAPSHOT
[INFO] --------------------------------[ jar ]---------------------------------
[INFO] 
[INFO] --- maven-dependency-plugin:2.8:tree (default-cli) @ spark-hello-world-scala ---
[INFO] spark-hello-world-scala:spark-hello-world-scala:jar:0.0.1-SNAPSHOT
[INFO] +- org.apache.spark:spark-core_2.12:jar:3.0.0:compile
[INFO] |  +- com.thoughtworks.paranamer:paranamer:jar:2.8:compile
[INFO] |  +- org.apache.avro:avro:jar:1.8.2:compile
[INFO] |  |  +- org.codehaus.jackson:jackson-core-asl:jar:1.9.13:compile
[INFO] |  |  +- org.codehaus.jackson:jackson-mapper-asl:jar:1.9.13:compile
[INFO] |  |  +- org.apache.commons:commons-compress:jar:1.8.1:compile
[INFO] |  |  \- org.tukaani:xz:jar:1.5:compile
[INFO] |  +- org.apache.avro:avro-mapred:jar:hadoop2:1.8.2:compile
[INFO] |  |  +- org.apache.avro:avro-ipc:jar:1.8.2:compile
[INFO] |  |  \- commons-codec:commons-codec:jar:1.9:compile
[INFO] |  +- com.twitter:chill_2.12:jar:0.9.5:compile
[INFO] |  |  \- com.esotericsoftware:kryo-shaded:jar:4.0.2:compile
[INFO] |  |     +- com.esotericsoftware:minlog:jar:1.3.0:compile
[INFO] |  |     \- org.objenesis:objenesis:jar:2.5.1:compile
[INFO] |  +- com.twitter:chill-java:jar:0.9.5:compile
[INFO] |  +- org.apache.xbean:xbean-asm7-shaded:jar:4.15:compile
[INFO] |  +- org.apache.hadoop:hadoop-client:jar:2.7.4:compile
[INFO] |  |  +- org.apache.hadoop:hadoop-common:jar:2.7.4:compile
[INFO] |  |  |  +- commons-cli:commons-cli:jar:1.2:compile
[INFO] |  |  |  +- xmlenc:xmlenc:jar:0.52:compile
[INFO] |  |  |  +- commons-httpclient:commons-httpclient:jar:3.1:compile
[INFO] |  |  |  +- commons-collections:commons-collections:jar:3.2.2:compile
[INFO] |  |  |  +- org.mortbay.jetty:jetty-sslengine:jar:6.1.26:compile
[INFO] |  |  |  +- javax.servlet.jsp:jsp-api:jar:2.1:runtime
[INFO] |  |  |  +- commons-configuration:commons-configuration:jar:1.6:compile
[INFO] |  |  |  |  \- commons-digester:commons-digester:jar:1.8:compile
[INFO] |  |  |  |     \- commons-beanutils:commons-beanutils:jar:1.7.0:compile
[INFO] |  |  |  +- com.google.code.gson:gson:jar:2.2.4:compile
[INFO] |  |  |  +- org.apache.hadoop:hadoop-auth:jar:2.7.4:compile
[INFO] |  |  |  |  +- org.apache.httpcomponents:httpclient:jar:4.2.5:compile
[INFO] |  |  |  |  |  \- org.apache.httpcomponents:httpcore:jar:4.2.4:compile
[INFO] |  |  |  |  \- org.apache.directory.server:apacheds-kerberos-codec:jar:2.0.0-M15:compile
[INFO] |  |  |  |     +- org.apache.directory.server:apacheds-i18n:jar:2.0.0-M15:compile
[INFO] |  |  |  |     +- org.apache.directory.api:api-asn1-api:jar:1.0.0-M20:compile
[INFO] |  |  |  |     \- org.apache.directory.api:api-util:jar:1.0.0-M20:compile
[INFO] |  |  |  +- org.apache.curator:curator-client:jar:2.7.1:compile
[INFO] |  |  |  \- org.apache.htrace:htrace-core:jar:3.1.0-incubating:compile
[INFO] |  |  +- org.apache.hadoop:hadoop-hdfs:jar:2.7.4:compile
[INFO] |  |  |  +- org.mortbay.jetty:jetty-util:jar:6.1.26:compile
[INFO] |  |  |  \- xerces:xercesImpl:jar:2.9.1:compile
[INFO] |  |  |     \- xml-apis:xml-apis:jar:1.3.04:compile
[INFO] |  |  +- org.apache.hadoop:hadoop-mapreduce-client-app:jar:2.7.4:compile
[INFO] |  |  |  +- org.apache.hadoop:hadoop-mapreduce-client-common:jar:2.7.4:compile
[INFO] |  |  |  |  +- org.apache.hadoop:hadoop-yarn-client:jar:2.7.4:compile
[INFO] |  |  |  |  \- org.apache.hadoop:hadoop-yarn-server-common:jar:2.7.4:compile
[INFO] |  |  |  \- org.apache.hadoop:hadoop-mapreduce-client-shuffle:jar:2.7.4:compile
[INFO] |  |  +- org.apache.hadoop:hadoop-yarn-api:jar:2.7.4:compile
[INFO] |  |  +- org.apache.hadoop:hadoop-mapreduce-client-core:jar:2.7.4:compile
[INFO] |  |  |  \- org.apache.hadoop:hadoop-yarn-common:jar:2.7.4:compile
[INFO] |  |  |     +- javax.xml.bind:jaxb-api:jar:2.2.2:compile
[INFO] |  |  |     |  \- javax.xml.stream:stax-api:jar:1.0-2:compile
[INFO] |  |  |     +- org.codehaus.jackson:jackson-jaxrs:jar:1.9.13:compile
[INFO] |  |  |     \- org.codehaus.jackson:jackson-xc:jar:1.9.13:compile
[INFO] |  |  +- org.apache.hadoop:hadoop-mapreduce-client-jobclient:jar:2.7.4:compile
[INFO] |  |  \- org.apache.hadoop:hadoop-annotations:jar:2.7.4:compile
[INFO] |  +- org.apache.spark:spark-launcher_2.12:jar:3.0.0:compile
[INFO] |  +- org.apache.spark:spark-kvstore_2.12:jar:3.0.0:compile
[INFO] |  |  +- org.fusesource.leveldbjni:leveldbjni-all:jar:1.8:compile
[INFO] |  |  +- com.fasterxml.jackson.core:jackson-core:jar:2.10.0:compile
[INFO] |  |  \- com.fasterxml.jackson.core:jackson-annotations:jar:2.10.0:compile
[INFO] |  +- org.apache.spark:spark-network-common_2.12:jar:3.0.0:compile
[INFO] |  +- org.apache.spark:spark-network-shuffle_2.12:jar:3.0.0:compile
[INFO] |  +- org.apache.spark:spark-unsafe_2.12:jar:3.0.0:compile
[INFO] |  +- javax.activation:activation:jar:1.1.1:compile
[INFO] |  +- org.apache.curator:curator-recipes:jar:2.7.1:compile
[INFO] |  |  +- org.apache.curator:curator-framework:jar:2.7.1:compile
[INFO] |  |  \- com.google.guava:guava:jar:16.0.1:compile
[INFO] |  +- org.apache.zookeeper:zookeeper:jar:3.4.14:compile
[INFO] |  |  \- org.apache.yetus:audience-annotations:jar:0.5.0:compile
[INFO] |  +- javax.servlet:javax.servlet-api:jar:3.1.0:compile
[INFO] |  +- org.apache.commons:commons-lang3:jar:3.9:compile
[INFO] |  +- org.apache.commons:commons-math3:jar:3.4.1:compile
[INFO] |  +- org.apache.commons:commons-text:jar:1.6:compile
[INFO] |  +- com.google.code.findbugs:jsr305:jar:3.0.0:compile
[INFO] |  +- org.slf4j:slf4j-api:jar:1.7.30:compile
[INFO] |  +- org.slf4j:jul-to-slf4j:jar:1.7.30:compile
[INFO] |  +- org.slf4j:jcl-over-slf4j:jar:1.7.30:compile
[INFO] |  +- log4j:log4j:jar:1.2.17:compile
[INFO] |  +- org.slf4j:slf4j-log4j12:jar:1.7.30:compile
[INFO] |  +- com.ning:compress-lzf:jar:1.0.3:compile
[INFO] |  +- org.xerial.snappy:snappy-java:jar:1.1.7.5:compile
[INFO] |  +- org.lz4:lz4-java:jar:1.7.1:compile
[INFO] |  +- com.github.luben:zstd-jni:jar:1.4.4-3:compile
[INFO] |  +- org.roaringbitmap:RoaringBitmap:jar:0.7.45:compile
[INFO] |  |  \- org.roaringbitmap:shims:jar:0.7.45:compile
[INFO] |  +- commons-net:commons-net:jar:3.1:compile
[INFO] |  +- org.scala-lang.modules:scala-xml_2.12:jar:1.2.0:compile
[INFO] |  +- org.scala-lang:scala-reflect:jar:2.12.10:compile
[INFO] |  +- org.json4s:json4s-jackson_2.12:jar:3.6.6:compile
[INFO] |  |  \- org.json4s:json4s-core_2.12:jar:3.6.6:compile
[INFO] |  |     +- org.json4s:json4s-ast_2.12:jar:3.6.6:compile
[INFO] |  |     \- org.json4s:json4s-scalap_2.12:jar:3.6.6:compile
[INFO] |  +- org.glassfish.jersey.core:jersey-client:jar:2.30:compile
[INFO] |  |  +- jakarta.ws.rs:jakarta.ws.rs-api:jar:2.1.6:compile
[INFO] |  |  \- org.glassfish.hk2.external:jakarta.inject:jar:2.6.1:compile
[INFO] |  +- org.glassfish.jersey.core:jersey-common:jar:2.30:compile
[INFO] |  |  +- jakarta.annotation:jakarta.annotation-api:jar:1.3.5:compile
[INFO] |  |  \- org.glassfish.hk2:osgi-resource-locator:jar:1.0.3:compile
[INFO] |  +- org.glassfish.jersey.core:jersey-server:jar:2.30:compile
[INFO] |  |  +- org.glassfish.jersey.media:jersey-media-jaxb:jar:2.30:compile
[INFO] |  |  \- jakarta.validation:jakarta.validation-api:jar:2.0.2:compile
[INFO] |  +- org.glassfish.jersey.containers:jersey-container-servlet:jar:2.30:compile
[INFO] |  +- org.glassfish.jersey.containers:jersey-container-servlet-core:jar:2.30:compile
[INFO] |  +- org.glassfish.jersey.inject:jersey-hk2:jar:2.30:compile
[INFO] |  |  +- org.glassfish.hk2:hk2-locator:jar:2.6.1:compile
[INFO] |  |  |  +- org.glassfish.hk2.external:aopalliance-repackaged:jar:2.6.1:compile
[INFO] |  |  |  +- org.glassfish.hk2:hk2-api:jar:2.6.1:compile
[INFO] |  |  |  \- org.glassfish.hk2:hk2-utils:jar:2.6.1:compile
[INFO] |  |  \- org.javassist:javassist:jar:3.25.0-GA:compile
[INFO] |  +- io.netty:netty-all:jar:4.1.47.Final:compile
[INFO] |  +- com.clearspring.analytics:stream:jar:2.9.6:compile
[INFO] |  +- io.dropwizard.metrics:metrics-core:jar:4.1.1:compile
[INFO] |  +- io.dropwizard.metrics:metrics-jvm:jar:4.1.1:compile
[INFO] |  +- io.dropwizard.metrics:metrics-json:jar:4.1.1:compile
[INFO] |  +- io.dropwizard.metrics:metrics-graphite:jar:4.1.1:compile
[INFO] |  +- io.dropwizard.metrics:metrics-jmx:jar:4.1.1:compile
[INFO] |  +- com.fasterxml.jackson.core:jackson-databind:jar:2.10.0:compile
[INFO] |  +- com.fasterxml.jackson.module:jackson-module-scala_2.12:jar:2.10.0:compile
[INFO] |  |  \- com.fasterxml.jackson.module:jackson-module-paranamer:jar:2.10.0:compile
[INFO] |  +- org.apache.ivy:ivy:jar:2.4.0:compile
[INFO] |  +- oro:oro:jar:2.0.8:compile
[INFO] |  +- net.razorvine:pyrolite:jar:4.30:compile
[INFO] |  +- net.sf.py4j:py4j:jar:0.10.9:compile
[INFO] |  +- org.apache.spark:spark-tags_2.12:jar:3.0.0:compile
[INFO] |  +- org.apache.commons:commons-crypto:jar:1.0.0:compile
[INFO] |  \- org.spark-project.spark:unused:jar:1.0.0:compile
[INFO] +- org.apache.spark:spark-mllib_2.12:jar:3.0.0:runtime
[INFO] |  +- org.scala-lang.modules:scala-parser-combinators_2.12:jar:1.1.2:compile
[INFO] |  +- org.apache.spark:spark-streaming_2.12:jar:3.0.0:runtime
[INFO] |  +- org.apache.spark:spark-graphx_2.12:jar:3.0.0:runtime
[INFO] |  |  +- com.github.fommil.netlib:core:jar:1.1.2:runtime
[INFO] |  |  \- net.sourceforge.f2j:arpack_combined_all:jar:0.1:runtime
[INFO] |  +- org.apache.spark:spark-mllib-local_2.12:jar:3.0.0:runtime
[INFO] |  +- org.scalanlp:breeze_2.12:jar:1.0:runtime
[INFO] |  |  +- org.scalanlp:breeze-macros_2.12:jar:1.0:runtime
[INFO] |  |  +- net.sf.opencsv:opencsv:jar:2.3:runtime
[INFO] |  |  +- com.github.wendykierp:JTransforms:jar:3.1:runtime
[INFO] |  |  |  \- pl.edu.icm:JLargeArrays:jar:1.5:runtime
[INFO] |  |  +- com.chuusai:shapeless_2.12:jar:2.3.3:runtime
[INFO] |  |  |  \- org.typelevel:macro-compat_2.12:jar:1.1.1:runtime
[INFO] |  |  +- org.typelevel:spire_2.12:jar:0.17.0-M1:runtime
[INFO] |  |  |  +- org.typelevel:spire-macros_2.12:jar:0.17.0-M1:runtime
[INFO] |  |  |  +- org.typelevel:spire-platform_2.12:jar:0.17.0-M1:runtime
[INFO] |  |  |  +- org.typelevel:spire-util_2.12:jar:0.17.0-M1:runtime
[INFO] |  |  |  +- org.typelevel:machinist_2.12:jar:0.6.8:runtime
[INFO] |  |  |  \- org.typelevel:algebra_2.12:jar:2.0.0-M2:runtime
[INFO] |  |  |     \- org.typelevel:cats-kernel_2.12:jar:2.0.0-M4:runtime
[INFO] |  |  \- org.scala-lang.modules:scala-collection-compat_2.12:jar:2.1.1:runtime
[INFO] |  \- org.glassfish.jaxb:jaxb-runtime:jar:2.3.2:runtime
[INFO] |     +- jakarta.xml.bind:jakarta.xml.bind-api:jar:2.3.2:runtime
[INFO] |     \- com.sun.istack:istack-commons-runtime:jar:3.0.8:runtime
[INFO] +- org.scala-lang:scala-library:jar:2.12.8:compile
[INFO] +- org.apache.spark:spark-sql_2.12:jar:3.0.0:compile
[INFO] |  +- com.univocity:univocity-parsers:jar:2.8.3:compile
[INFO] |  +- org.apache.spark:spark-sketch_2.12:jar:3.0.0:compile
[INFO] |  +- org.apache.spark:spark-catalyst_2.12:jar:3.0.0:compile
[INFO] |  |  +- org.codehaus.janino:janino:jar:3.0.16:compile
[INFO] |  |  +- org.codehaus.janino:commons-compiler:jar:3.0.16:compile
[INFO] |  |  +- org.antlr:antlr4-runtime:jar:4.7.1:compile
[INFO] |  |  \- org.apache.arrow:arrow-vector:jar:0.15.1:compile
[INFO] |  |     +- org.apache.arrow:arrow-format:jar:0.15.1:compile
[INFO] |  |     +- org.apache.arrow:arrow-memory:jar:0.15.1:compile
[INFO] |  |     \- com.google.flatbuffers:flatbuffers-java:jar:1.9.0:compile
[INFO] |  +- org.apache.orc:orc-core:jar:1.5.10:compile
[INFO] |  |  +- org.apache.orc:orc-shims:jar:1.5.10:compile
[INFO] |  |  +- com.google.protobuf:protobuf-java:jar:2.5.0:compile
[INFO] |  |  +- commons-lang:commons-lang:jar:2.6:compile
[INFO] |  |  +- io.airlift:aircompressor:jar:0.10:compile
[INFO] |  |  \- org.threeten:threeten-extra:jar:1.5.0:compile
[INFO] |  +- org.apache.orc:orc-mapreduce:jar:1.5.10:compile
[INFO] |  +- org.apache.hive:hive-storage-api:jar:2.7.1:compile
[INFO] |  +- org.apache.parquet:parquet-column:jar:1.10.1:compile
[INFO] |  |  +- org.apache.parquet:parquet-common:jar:1.10.1:compile
[INFO] |  |  \- org.apache.parquet:parquet-encoding:jar:1.10.1:compile
[INFO] |  \- org.apache.parquet:parquet-hadoop:jar:1.10.1:compile
[INFO] |     +- org.apache.parquet:parquet-format:jar:2.4.0:compile
[INFO] |     \- org.apache.parquet:parquet-jackson:jar:1.10.1:compile
[INFO] +- com.databricks:spark-xml_2.12:jar:0.9.0:compile
[INFO] |  +- commons-io:commons-io:jar:2.6:compile
[INFO] |  \- org.glassfish.jaxb:txw2:jar:2.3.2:compile
[INFO] +- org.apache.spark:spark-avro_2.12:jar:3.0.0:compile
[INFO] +- org.scalatest:scalatest_2.12:jar:3.0.5:test
[INFO] |  \- org.scalactic:scalactic_2.12:jar:3.0.5:test
[INFO] +- org.specs2:specs2-core_2.12:jar:4.2.0:test
[INFO] |  +- org.specs2:specs2-matcher_2.12:jar:4.2.0:test
[INFO] |  |  \- org.specs2:specs2-common_2.12:jar:4.2.0:test
[INFO] |  |     \- org.specs2:specs2-fp_2.12:jar:4.2.0:test
[INFO] |  \- org.scala-sbt:test-interface:jar:1.0:test
[INFO] +- org.specs2:specs2-junit_2.12:jar:4.2.0:test
[INFO] |  \- junit:junit:jar:4.12:test
[INFO] |     \- org.hamcrest:hamcrest-core:jar:1.3:test
[INFO] +- org.apache.logging.log4j:log4j-api:jar:2.13.3:compile
[INFO] +- org.apache.logging.log4j:log4j-core:jar:2.13.3:compile
[INFO] +- org.apache.logging.log4j:log4j-api-scala_2.12:jar:12.0:compile
[INFO] +- org.apache.logging.log4j:log4j-api:test-jar:tests:2.13.3:test
[INFO] \- org.apache.logging.log4j:log4j-to-slf4j:jar:2.13.3:compile