Ruby: generate require_relative calls for imported types

... instead of expecting the calling code to load everything that the
generated modules need.

Fixes these tests for Ruby:

* ImportsCastToImported
* ImportsCastToImported2
* ImportsParamsDefEnumImported
* ImportsParamsDefUsertypeImported

We intentionally use `require_relative` (instead of `require` as usual)
and only for imported types (not opaque types) - see the added code
comment in `RubyCompiler`'s implementation of `externalTypeDeclaration`
for more details.

Until now, we've been kind of living under the impression that the
generated Ruby files don't need explicit imports, because until
recently, all of our import-related tests in
https://github.com/kaitai-io/kaitai_struct_tests were passing despite
the fact that KSC has never generated any `require`s in Ruby. However,
it turned out that this was just a coincidence described in
kaitai-io/kaitai_struct#1099, caused by the
fact that the Ruby's `require` pollutes the global namespace and that
the imported specs for which we were missing `require`s were already
required by some other test. This was not the case in the 4 tests that
this commit fixes as mentioned above, which were added recently and all
import a format spec that no other test uses.

This "missing `require`s" problem would have been an issue in
[KSV](https://github.com/kaitai-io/kaitai_struct_visualizer) too, but
KSV works around it by requiring every single `*.rb` file that KSC has
generated (which will include any imported .ksy specs), and opaque types
must be explicitly imported using the `-r` command-line option. This
should still work after this commit, although calling `require` with
each `*.rb` file in the directory of generated format modules becomes
unnecessary.

Author: generalmimon

shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala

@@ -30,8 +30,8 @@ class ClassCompiler(
   }
 
   def compileExternalTypes(topClass: ClassSpec) = {
-    TypeProcessor.getExternalTypes(topClass).foreach((name) =>
-      lang.externalTypeDeclaration(name)
+    TypeProcessor.getExternalTypes(topClass).foreach((extType) =>
+      lang.externalTypeDeclaration(extType)
     )
   }
 

shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala

@@ -6,9 +6,22 @@ import io.kaitai.struct.format._
 
 import scala.collection.mutable
 
+sealed abstract class ExternalType {
+  def name: List[String]
+  def isOpaque: Boolean
+}
+case class ExternalUserType(classSpec: ClassSpec) extends ExternalType {
+  def name: List[String] = classSpec.name
+  def isOpaque: Boolean = classSpec.meta.isOpaque
+}
+case class ExternalEnum(enumSpec: EnumSpec) extends ExternalType {
+  def name: List[String] = enumSpec.name
+  def isOpaque: Boolean = false
+}
+
 object TypeProcessor {
-  def getExternalTypes(curClass: ClassSpec): Iterable[List[String]] = {
-    val res = mutable.Set[List[String]]()
+  def getExternalTypes(curClass: ClassSpec): Iterable[ExternalType] = {
+    val res = mutable.Set[ExternalType]()
     curClass.seq.foreach((attr) =>
       res ++= getExternalTypesFromDataType(attr.dataType, curClass)
     )
@@ -31,17 +44,17 @@ object TypeProcessor {
     res
   }
 
-  def getExternalTypesFromDataType(dataType: DataType, curClass: ClassSpec): Iterable[List[String]] = {
+  def getExternalTypesFromDataType(dataType: DataType, curClass: ClassSpec): Iterable[ExternalType] = {
     dataType match {
       case ut: UserType =>
         if (ut.isExternal(curClass)) {
-          List(ut.classSpec.get.name)
+          List(ExternalUserType(ut.classSpec.get))
         } else {
           List()
         }
       case et: EnumType =>
         if (et.isExternal(curClass)) {
-          List(et.enumSpec.get.name)
+          List(ExternalEnum(et.enumSpec.get))
         } else {
           List()
         }

shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala

@@ -118,8 +118,8 @@ class CppCompiler(
     }
   }
 
-  override def externalTypeDeclaration(name: List[String]): Unit =
-    importListHdr.addLocal(outFileNameHeader(name.head))
+  override def externalTypeDeclaration(extType: ExternalType): Unit =
+    importListHdr.addLocal(outFileNameHeader(extType.name.head))
 
   override def classHeader(name: List[String]): Unit = {
     val className = types2class(List(name.last))

shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala

@@ -7,7 +7,7 @@ import io.kaitai.struct.exprlang.Ast.expr
 import io.kaitai.struct.format._
 import io.kaitai.struct.languages.components._
 import io.kaitai.struct.translators.JavaScriptTranslator
-import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils}
+import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils, ExternalType}
 
 class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   extends LanguageCompiler(typeProvider, config)
@@ -58,8 +58,8 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts("});")
   }
 
-  override def externalTypeDeclaration(name: List[String]): Unit = {
-    val className = type2class(name.head)
+  override def externalTypeDeclaration(extType: ExternalType): Unit = {
+    val className = type2class(extType.name.head)
     importList.add(s"./$className")
   }
 

shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala

@@ -1,6 +1,6 @@
 package io.kaitai.struct.languages
 
-import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils}
+import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils, ExternalType}
 import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError, ValidationNotEqualError}
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.exprlang.Ast
@@ -32,8 +32,8 @@ class LuaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   override def outImports(topClass: ClassSpec) =
     importList.toList.mkString("", "\n", "\n")
 
-  override def externalTypeDeclaration(name: List[String]): Unit =
-    importList.add("require(\"" + name.head + "\")")
+  override def externalTypeDeclaration(extType: ExternalType): Unit =
+    importList.add("require(\"" + extType.name.head + "\")")
 
   override def fileHeader(topClassName: String): Unit = {
     outHeader.puts(s"-- $headerComment")

shared/src/main/scala/io/kaitai/struct/languages/NimCompiler.scala

@@ -6,7 +6,7 @@ import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format._
 import io.kaitai.struct.languages.components._
 import io.kaitai.struct.translators.NimTranslator
-import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils}
+import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils, ExternalType}
 
 class NimCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   extends LanguageCompiler(typeProvider, config)
@@ -54,8 +54,8 @@ class NimCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts
   }
 
-  override def externalTypeDeclaration(name: List[String]): Unit =
-    importList.add(name.head)
+  override def externalTypeDeclaration(extType: ExternalType): Unit =
+    importList.add(extType.name.head)
   override def innerEnums = false
   override val translator: NimTranslator = new NimTranslator(typeProvider, importList)
   override def universalFooter: Unit = {

shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala

@@ -8,6 +8,7 @@ import io.kaitai.struct.format._
 import io.kaitai.struct.languages.components.{ExceptionNames, _}
 import io.kaitai.struct.translators.{PerlTranslator, TypeProvider}
 import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig}
+import io.kaitai.struct.ExternalType
 
 class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   extends LanguageCompiler(typeProvider, config)
@@ -50,8 +51,8 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts("1;")
   }
 
-  override def externalTypeDeclaration(name: List[String]): Unit =
-    importList.add(type2class(name.head))
+  override def externalTypeDeclaration(extType: ExternalType): Unit =
+    importList.add(type2class(extType.name.head))
 
   override def classHeader(name: List[String]): Unit = {
     out.puts

shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala

@@ -7,7 +7,7 @@ import io.kaitai.struct.exprlang.Ast.expr
 import io.kaitai.struct.format._
 import io.kaitai.struct.languages.components._
 import io.kaitai.struct.translators.PythonTranslator
-import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, StringLanguageOutputWriter, Utils}
+import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, StringLanguageOutputWriter, Utils, ExternalType}
 
 class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   extends LanguageCompiler(typeProvider, config)
@@ -77,8 +77,8 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts
   }
 
-  override def externalTypeDeclaration(name: List[String]): Unit = {
-    val moduleName = name.head
+  override def externalTypeDeclaration(extType: ExternalType): Unit = {
+    val moduleName = extType.name.head
     importList.add(
       if (config.pythonPackage.nonEmpty) {
         s"from ${config.pythonPackage} import $moduleName"

shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala

@@ -7,7 +7,7 @@ import io.kaitai.struct.exprlang.Ast.expr
 import io.kaitai.struct.format._
 import io.kaitai.struct.languages.components._
 import io.kaitai.struct.translators.RubyTranslator
-import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils}
+import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils, ExternalType}
 
 class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   extends LanguageCompiler(typeProvider, config)
@@ -34,13 +34,13 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   override def indent: String = "  "
 
   override def outImports(topClass: ClassSpec) =
-    importList.toList.map((x) => s"require '$x'").mkString("\n") + "\n"
+    importList.toList.mkString("", "\n", "\n")
 
   override def fileHeader(topClassName: String): Unit = {
     outHeader.puts(s"# $headerComment")
     outHeader.puts
 
-    importList.add("kaitai/struct/struct")
+    importList.add("require 'kaitai/struct/struct'")
 
     out.puts
 
@@ -61,6 +61,28 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts
   }
 
+  override def externalTypeDeclaration(extType: ExternalType): Unit = {
+    // Do not generate imports for opaque types
+    if (extType.isOpaque)
+      return
+    // We intentionally use `require_relative` instead of `require`. This is
+    // because in our case, it's more reasonable to assume that the imported
+    // format module is in the same directory as the importing one (because KSC
+    // generates them to the same directory) than to assume that the imported
+    // format module is in `$LOAD_PATH`. In addition, since `$LOAD_PATH` is an
+    // array of paths and the first matching path wins with `require`, by
+    // avoiding `require` we eliminate the possibility that we load a different
+    // module than intended if the directory with compiled format modules is not
+    // the first path in `$LOAD_PATH`.
+    //
+    // However, we are careful not to generate any imports for opaque types.
+    // Opaque types are not generated by KSC, so there's no reason to assume
+    // that they are defined in .rb files in the same directory as the generated
+    // format module that uses them. Instead, we expect the user to make them
+    // globally available, and it's up to them how they do that.
+    importList.add(s"require_relative '${extType.name.head}'")
+  }
+
   override def classHeader(name: String): Unit = {
     out.puts(s"class ${type2class(name)} < $kstructName")
     out.inc
@@ -188,7 +210,7 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
         }
         s"$kstreamName::$procName($srcExpr, ${expression(xorValue)})"
       case ProcessZlib =>
-        importList.add("zlib")
+        importList.add("require 'zlib'")
         s"Zlib::Inflate.inflate($srcExpr)"
       case ProcessRotate(isLeft, rotValue) =>
         val expr = if (isLeft) {

shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala

@@ -6,7 +6,7 @@ import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format.{NoRepeat, RepeatEos, RepeatExpr, RepeatSpec, _}
 import io.kaitai.struct.languages.components._
 import io.kaitai.struct.translators.RustTranslator
-import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils}
+import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils, ExternalType}
 
 class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   extends LanguageCompiler(typeProvider, config)
@@ -54,9 +54,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts
   }
 
-  override def externalTypeDeclaration(name: List[String]): Unit = {
-    val className = type2class(name.last)
-    val pkg = type2classAbs(name)
+  override def externalTypeDeclaration(extType: ExternalType): Unit = {
+    val className = type2class(extType.name.last)
+    val pkg = type2classAbs(extType.name)
 
     importList.add(s"$pkg::$className")
   }

shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala

@@ -4,7 +4,7 @@ import io.kaitai.struct.datatype.{DataType, Endianness, FixedEndian, InheritedEn
 import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format._
 import io.kaitai.struct.translators.AbstractTranslator
-import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig}
+import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, ExternalType}
 
 import scala.collection.mutable.ListBuffer
 
@@ -81,7 +81,7 @@ abstract class LanguageCompiler(
     * type.
     * @param name absolute path to the type in KS notation (lower underscore)
     */
-  def externalTypeDeclaration(name: List[String]): Unit = {}
+  def externalTypeDeclaration(extType: ExternalType): Unit = {}
 
   def classDoc(name: List[String], doc: DocSpec): Unit = {}
   def classHeader(name: List[String]): Unit