Packages

t

org.apache.spark.sql

SparkSessionExtensionsProvider

trait SparkSessionExtensionsProvider extends (SparkSessionExtensions) => Unit

:: Unstable ::

Base trait for implementations used by SparkSessionExtensions

For example, now we have an external function named Age to register as an extension for SparkSession:

package org.apache.spark.examples.extensions

import org.apache.spark.sql.catalyst.expressions.{CurrentDate, Expression, RuntimeReplaceable, SubtractDates}

case class Age(birthday: Expression, child: Expression) extends RuntimeReplaceable {

  def this(birthday: Expression) = this(birthday, SubtractDates(CurrentDate(), birthday))
  override def exprsReplaced: Seq[Expression] = Seq(birthday)
  override protected def withNewChildInternal(newChild: Expression): Expression = copy(newChild)
}

We need to create our extension which inherits SparkSessionExtensionsProvider Example:

package org.apache.spark.examples.extensions

import org.apache.spark.sql.{SparkSessionExtensions, SparkSessionExtensionsProvider}
import org.apache.spark.sql.catalyst.FunctionIdentifier
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo}

class MyExtensions extends SparkSessionExtensionsProvider {
  override def apply(v1: SparkSessionExtensions): Unit = {
    v1.injectFunction(
      (new FunctionIdentifier("age"),
        new ExpressionInfo(classOf[Age].getName, "age"),
        (children: Seq[Expression]) => new Age(children.head)))
  }
}

Then, we can inject MyExtensions in three ways,

Annotations
@DeveloperApi() @Unstable() @Since("3.2.0")
Source
SparkSessionExtensionsProvider.scala
Since

3.2.0

See also

SparkSessionExtensions

SparkSession.Builder

java.util.ServiceLoader

Linear Supertypes
(SparkSessionExtensions) => Unit, AnyRef, Any

Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. SparkSessionExtensionsProvider
  2. Function1
  3. AnyRef
  4. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. Protected

Abstract Value Members

  1.  abstract def apply(v1: SparkSessionExtensions): Unit
    Definition Classes
    Function1

Concrete Value Members

  1.  final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2.  final def ##: Int
    Definition Classes
    AnyRef → Any
  3.  final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4.  def andThen[A](g: (Unit) => A): (SparkSessionExtensions) => A
    Definition Classes
    Function1
    Annotations
    @unspecialized()
  5.  final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  6.  def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.CloneNotSupportedException]) @IntrinsicCandidate() @native()
  7.  def compose[A](g: (A) => SparkSessionExtensions): (A) => Unit
    Definition Classes
    Function1
    Annotations
    @unspecialized()
  8.  final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  9.  def equals(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef → Any
  10.  final def getClass(): Class[_ <: AnyRef]
    Definition Classes
    AnyRef → Any
    Annotations
    @IntrinsicCandidate() @native()
  11.  def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @IntrinsicCandidate() @native()
  12.  final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  13.  final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  14.  final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @IntrinsicCandidate() @native()
  15.  final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @IntrinsicCandidate() @native()
  16.  final def synchronized[T0](arg0: => T0): T0
    Definition Classes
    AnyRef
  17.  def toString(): String
    Definition Classes
    Function1 → AnyRef → Any
  18.  final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  19.  final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException]) @native()
  20.  final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])

Deprecated Value Members

  1.  def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.Throwable]) @Deprecated
    Deprecated

    (Since version 9)

Inherited from (SparkSessionExtensions) => Unit

Inherited from AnyRef

Inherited from Any

Ungrouped