I am migrating a project from scala 2.12 to 2.13 and I manage to make everything to compile. Buy when I run my tests I get an error
com.alexm.spark.ml.bojo.StringValueImputerTest *** ABORTED ***
java.lang.RuntimeException: Unable to load a Suite class that was discovered in the runpath: com.alexm.spark.ml.bojo.StringValueImputerTest
at org.scalatest.tools.DiscoverySuite$.getSuiteInstance(DiscoverySuite.scala:81)
at org.scalatest.tools.DiscoverySuite.$anonfun$nestedSuites$1(DiscoverySuite.scala:38)
at scala.collection.StrictOptimizedIterableOps.map(StrictOptimizedIterableOps.scala:100)
at scala.collection.StrictOptimizedIterableOps.map$(StrictOptimizedIterableOps.scala:87)
at scala.collection.immutable.Vector.map(Vector.scala:113)
at org.scalatest.tools.DiscoverySuite.<init>(DiscoverySuite.scala:37)
at org.scalatest.tools.Runner$.genDiscoSuites$1(Runner.scala:1168)
at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:1253)
at org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24(Runner.scala:1033)
at org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24$adapted(Runner.scala:1011)
...
Cause: java.lang.NoClassDefFoundError: org/json4s/JsonAST$JValue
at org.apache.spark.ui.jobs.JobsTab.<init>(JobsTab.scala:47)
at org.apache.spark.ui.SparkUI.initialize(SparkUI.scala:81)
at org.apache.spark.ui.SparkUI.<init>(SparkUI.scala:103)
at org.apache.spark.ui.SparkUI$.create(SparkUI.scala:223)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:484)
at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2704)
at org.apache.spark.sql.SparkSession$Builder.$anonfun$getOrCreate$2(SparkSession.scala:953)
at scala.Option.getOrElse(Option.scala:202)
at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:947)
at com.alexm.spark.WithSpark$.getSpark(WithSpark.scala:116)
...
Cause: java.lang.ClassNotFoundException: org.json4s.JsonAST$JValue
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
at org.apache.spark.ui.jobs.JobsTab.<init>(JobsTab.scala:47)
at org.apache.spark.ui.SparkUI.initialize(SparkUI.scala:81)
at org.apache.spark.ui.SparkUI.<init>(SparkUI.scala:103)
at org.apache.spark.ui.SparkUI$.create(SparkUI.scala:223)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:484)
at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2704)
...
*** RUN ABORTED ***
java.lang.NoClassDefFoundError: org/json4s/JsonAST$JValue
at org.apache.spark.ui.jobs.JobsTab.<init>(JobsTab.scala:47)
at org.apache.spark.ui.SparkUI.initialize(SparkUI.scala:81)
at org.apache.spark.ui.SparkUI.<init>(SparkUI.scala:103)
at org.apache.spark.ui.SparkUI$.create(SparkUI.scala:223)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:484)
at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2704)
at org.apache.spark.sql.SparkSession$Builder.$anonfun$getOrCreate$2(SparkSession.scala:953)
at scala.Option.getOrElse(Option.scala:202)
at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:947)
at com.alexm.spark.WithSpark$.getSpark(WithSpark.scala:116)
...
Cause: java.lang.ClassNotFoundException: org.json4s.JsonAST$JValue
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
at org.apache.spark.ui.jobs.JobsTab.<init>(JobsTab.scala:47)
at org.apache.spark.ui.SparkUI.initialize(SparkUI.scala:81)
at org.apache.spark.ui.SparkUI.<init>(SparkUI.scala:103)
at org.apache.spark.ui.SparkUI$.create(SparkUI.scala:223)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:484)
at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2704)
...
my pom file is
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<profiles>
<profile>
<id>2.11</id>
<properties>
<scala.major>2.11</scala.major>
<scala.minor>8</scala.minor>
<scala.version>${scala.major}.${scala.minor}</scala.version>
<spark.version>2.3.1</spark.version>
<atl.version>0.2.4</atl.version>
</properties>
</profile>
<profile>
<id>2.12</id>
<properties>
<scala.major>2.12</scala.major>
<scala.minor>10</scala.minor>
<scala.version>${scala.major}.${scala.minor}</scala.version>
<spark.version>3.0.0</spark.version>
<atl.version>0.2.4</atl.version>
</properties>
</profile>
<profile>
<id>2.13</id>
<properties>
<scala.major>2.13</scala.major>
<scala.minor>8</scala.minor>
<scala.version>${scala.major}.${scala.minor}</scala.version>
<spark.version>3.3.0</spark.version>
<atl.version>1.0.4</atl.version>
</properties>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
</profile>
</profiles>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.testSource>1.8</maven.compiler.testSource>
<maven.compiler.testTarget>1.8</maven.compiler.testTarget>
<jersey.version>2.22.2</jersey.version>
<jar_path>${project.build.outputDirectory}/${project.build.finalName}</jar_path>
<!--this property is needed by javax.ws.rs-api 2.1-->
<packaging.type>jar</packaging.type>
<skipTests>false</skipTests>
</properties>
<groupId>com.alexm.spark.ml</groupId>
<artifactId>featureTransformations_${scala.major}_${spark.version}</artifactId>
<version>1.12-SNAPSHOT</version>
<packaging>jar</packaging>
<description>Spark application</description>
<organization>
<name>Model Building</name>
<url>https://office.alexm.com/staff/team.html?orgunit_id=60004272</url>
</organization>
<distributionManagement>
<snapshotRepository>
<id>maven-alexm-snapshots</id>
<name>alexm Snapshot Repository</name>
<url>https://jfrog.alexm.com:443/artifactory/maven-alexm-snapshots</url>
</snapshotRepository>
<repository>
<id>maven-alexm-releases</id>
<name>alexm Release Repository</name>
<url>https://jfrog.alexm.com/artifactory/maven-alexm-releases</url>
</repository>
</distributionManagement>
<scm>
<developerConnection>scm:git:[email protected]:core/feature-engineering/model-building/feature-transformations.git</developerConnection>
<tag>featureTransformations-1.2</tag>
</scm>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.major}</artifactId>
<version>3.0.8</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.alexm.atl</groupId>
<artifactId>atl_${scala.major}</artifactId>
<version>${atl.version}</version>
<exclusions>
<exclusion>
<groupId>org.json4s</groupId>
<artifactId>json4s-jackson_${scala.major}</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.major}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.12</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.major}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.major}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_${scala.major}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.6.1</version>
<executions>
<execution>
<phase>compile</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.3.2</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>jar</goal>
</goals>
<configuration>
<classifier>nodeps</classifier>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>3.0.1</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
<version>2.5.3</version>
<configuration>
<autoVersionSubmodules>false</autoVersionSubmodules>
<useReleaseProfile>false</useReleaseProfile>
<preparationGoals>clean test</preparationGoals>
<scmCommentPrefix>[maven-release-plugin][ci skip]</scmCommentPrefix>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>versions-maven-plugin</artifactId>
<version>2.3</version>
<executions>
<execution>
<id>package</id>
<phase>package</phase>
<goals>
<goal>display-dependency-updates</goal>
<goal>display-parent-updates</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.7</version>
<configuration>
<skipTests>true</skipTests>
</configuration>
</plugin>
<plugin>
<groupId>org.scoverage</groupId>
<artifactId>scoverage-maven-plugin</artifactId>
<version>1.3.0</version>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.1.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>1.0</version>
<configuration>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<filereports>WDF TestSuite.txt</filereports>
<skipTests>${skipTests}</skipTests>
</configuration>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.1</version>
<executions>
<execution>
<id>scala-compile-first</id>
<phase>process-resources</phase>
<goals>
<goal>add-source</goal>
<goal>compile</goal>
</goals>
</execution>
<execution>
<id>scala-test-compile</id>
<phase>process-test-resources</phase>
<goals>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
<configuration>
<scalaCompatVersion>${scala.major}</scalaCompatVersion>
<scalaVersion>${scala.version}</scalaVersion>
</configuration>
</plugin>
</plugins>
</build>
</project>
Can you explain this kind of problems? How can a class be found at the runpath but not be loaded. Does it mean that it is not at the classpath? Do I miss something from my pom?