我有以下可以编译的函数。
def compare(dbo: Dataset[Cols], ods: Dataset[Cols]) = {
val j = dbo.crossJoin(ods)
// Tried val j = dbo.joinWith(ods, func.expr("true")) too
j.take(5).foreach(r => println(r))
}
但是在提交到Spark时出现了运行时错误。
连接条件缺失或微不足道。(如果使用joinWith而不是crossJoin,请使用CROSS JOIN语法允许这些关系之间的笛卡尔积。);在org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts$$anonfun$apply$21.applyOrElse(Optimizer.scala:1067);在org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts$$anonfun$apply$21.applyOrElse(Optimizer.scala:1064);在org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$2.apply(TreeNode.scala:268);在org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$2.apply(TreeNode.scala:268);在org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:70);在org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:267);在org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:273);在org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:273);在org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:307);在org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:188);在org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:305);在org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:273);在org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:273);在org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:273);在org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:307);在org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:188);在org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:305);在org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:273);在org.apache.spark.sql.catalyst.trees.TreeNode.transform(TreeNode.scala:257);在org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts.apply(Optimizer.scala:1064);在org.apache.spark.sql.catalyst.optimizer.CheckCartesianProducts.apply(Optimizer.scala:1049);在org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:85);在org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:82);在scala.collection.IndexedSeqOptimized$class.foldl(IndexedSeqOptimized.scala:57);在scala.collection.IndexedSeqOptimized$class.foldLeft(IndexedSeqOptimized.scala:66);在scala.collection.mutable.WrappedArray.foldLeft(WrappedArray.scala:35);在org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:82);在org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:74);在scala.collection.immutable.List.foreach(List.scala:381);在org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:74);在org.apache.spark.sql.execution.QueryExecution.optimizedPlan$lzycompute(QueryExecution.scala:78);在org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:78);在org.apache.spark.sql.execution.QueryExecution.sparkPlan$lzycompute(QueryExecution.scala:84);在org.apache.spark.sql.execution.QueryExecution.sparkPlan(QueryExecution.scala:80);在org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:89);在org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:89);在org.apache.spark.sql.Dataset.withTypedCallback(Dataset.scala:2814);在org.apache.spark.sql.Dataset.head(Dataset.scala:2127);在org.apache.spark.sql.Dataset.take(Dataset.scala:2342);在MappingPoint$.compare(MappingPoint.scala:43);在MappingPoint$.main(MappingPoint.scala:33);在MappingPoint.main(MappingPoint.scala);在sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method);在sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source);在sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source);在java.lang.reflect.Method
PATH
中。是时候升级了。至少你会知道我们是否正在与已经修复的问题作斗争(这样你就不会浪费时间)。 - Jacek Laskowski