问题
I'm seeking a method to create a serializable TypeTag without using compile time tools (completely relying on runtime). This is a basic feature for all reflective language.
The answer in this post proposed a few methods:
In Scala, how to create a TypeTag from a type that is serializable?
NONE OF THEM WORKED:
package com.tribbloids.spike.scala_spike
import java.io.{
ByteArrayInputStream,
ByteArrayOutputStream,
ObjectInputStream,
ObjectOutputStream
}
import org.apache.spark.sql.catalyst.ScalaReflection
import org.apache.spark.sql.catalyst.ScalaReflection.universe
import org.scalatest.FunSpec
class TypeTagFromType extends FunSpec {
import ScalaReflection.universe._
it("create TypeTag from reflection") {
val ttg = typeTag[String]
val ttg2 = TypeUtils.createTypeTag_fast(ttg.tpe, ttg.mirror)
val ttg3 = TypeUtils.createTypeTag_slow(ttg.tpe, ttg.mirror)
Seq(
ttg -> "from static inference",
ttg2 -> "from dynamic type - fast",
ttg3 -> "from dynamic type - slow"
).map {
case (tt, k) =>
println(k)
try {
val bytes = serialise(tt)
val tt2 = deserialise(bytes)
assert(tt.tpe =:= tt2.tpe)
} catch {
case e: Throwable =>
e.printStackTrace()
}
}
}
def serialise(tt: universe.TypeTag[_]): Array[Byte] = {
val bos = new ByteArrayOutputStream()
try {
val out = new ObjectOutputStream(bos)
out.writeObject(tt)
out.flush()
val array = bos.toByteArray
array
} finally {
bos.close()
}
}
def deserialise(tt: Array[Byte]): TypeTag[_] = {
val bis = new ByteArrayInputStream(tt)
try {
val in = new ObjectInputStream(bis)
in.readObject().asInstanceOf[TypeTag[_]]
} finally {
bis.close()
}
}
}
object TypeUtils {
import ScalaReflection.universe._
def createTypeTag_fast[T](
tpe: Type,
mirror: Mirror
): TypeTag[T] = {
TypeTag.apply(
mirror,
NaiveTypeCreator(tpe)
)
}
def createTypeTag_slow[T](
tpe: Type,
mirror: Mirror
): TypeTag[T] = {
val toolbox = scala.tools.reflect.ToolBox(mirror).mkToolBox()
val tree = toolbox.parse(s"scala.reflect.runtime.universe.typeTag[$tpe]")
val result = toolbox.eval(tree).asInstanceOf[TypeTag[T]]
result
}
case class NaiveTypeCreator(tpe: Type) extends reflect.api.TypeCreator {
def apply[U <: reflect.api.Universe with Singleton](
m: reflect.api.Mirror[U]): U#Type = {
// assert(m eq mirror, s"TypeTag[$tpe] defined in $mirror cannot be migrated to $m.")
tpe.asInstanceOf[U#Type]
}
}
}
For ttg2 and ttg3 created in runtime, An error is encountered when serializing or deserializing them, ttg2 encounter the error:
java.io.NotSerializableException: scala.reflect.runtime.JavaMirrors$JavaMirror$$anon$2
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1184)
at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348)
...
ttg3 encounter the error:
java.lang.ClassNotFoundException: __wrapper$1$71de08de01364321af52d1563247025d.__wrapper$1$71de08de01364321af52d1563247025d$$typecreator1$1
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:348)
at java.io.ObjectInputStream.resolveClass(ObjectInputStream.java:686)
at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1868)
at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1751)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2042)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1573)
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2287)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2211)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2069)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1573)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:431)
...
If you are familiar with the design of scala reflection, can you give a 'canonical' implementation that produce a functioning TypeTag?
来源：https://stackoverflow.com/questions/59473734/in-scala-2-12-why-none-of-the-typetag-created-in-runtime-is-serializable