org.apache.spark.streaming.aliyun.ons

OnsUtils

object OnsUtils

Linear Supertypes
AnyRef, Any
Ordering
  1. Alphabetic
  2. By inheritance
Inherited
  1. OnsUtils
  2. AnyRef
  3. Any
  1. Hide All
  2. Show all
Learn more about member selection
Visibility
  1. Public
  2. All

Value Members

  1. final def !=(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef
  2. final def !=(arg0: Any): Boolean

    Definition Classes
    Any
  3. final def ##(): Int

    Definition Classes
    AnyRef → Any
  4. final def ==(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef
  5. final def ==(arg0: Any): Boolean

    Definition Classes
    Any
  6. final def asInstanceOf[T0]: T0

    Definition Classes
    Any
  7. def clone(): AnyRef

    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  8. def createStream(jssc: JavaStreamingContext, consumerId: String, topic: String, tags: String, accessKeyId: String, accessKeySecret: String, storageLevel: StorageLevel, func: Function[Message, Array[Byte]]): JavaReceiverInputDStream[Array[Byte]]

    Create an input stream that pulls message from a Aliyun ONS stream.

    Create an input stream that pulls message from a Aliyun ONS stream.

    JavaStreamingContext jssc = ...
    String cid = "ConsumerID"
    String topic = "sample-topic"
    String subExpression = "*"
    String accessKeyId = "kj7aY*******UYx6"
    String accessKeySecret = "AiNMAlxz*************1PxaPaL8t"
    String accessKeySecret = ""
    
    static class ReadMessage implements Function<Message, Byte[]> {
        @Override
        public Byte[] call(Message msg) {
            return msg.getBody;
    }
    
    JavaReceiverInputDStream<Byte[]> onsStream = OnsUtils.createStream(
        ssc,
        cId,
        topic,
        subExpression,
        accessKeyId,
        accessKeySecret,
        StorageLevel.MEMORY_AND_DISK_2,
        ReadMessage)
    
     onsStream.foreachRDD(rdd => {
       ...
     })
    jssc

    Java streamingContext object

    consumerId

    Name of a set of consumers

    topic

    Which topic to subscribe

    tags

    Which tag to subscribe

    accessKeyId

    Aliyun Access Key ID

    accessKeySecret

    Aliyun Access Key Secret

    storageLevel

    Storage level to use for storing the received objects. StorageLevel.MEMORY_AND_DISK_2 is recommended.

    func

    Extract information from ONS message

    returns

    Annotations
    @Experimental()
  9. def createStream(ssc: StreamingContext, consumerId: String, topic: String, tags: String, accessKeyId: String, accessKeySecret: String, storageLevel: StorageLevel, func: (Message) ⇒ Array[Byte]): ReceiverInputDStream[Array[Byte]]

    Create an input stream that pulls message from a Aliyun ONS stream.

    Create an input stream that pulls message from a Aliyun ONS stream.

    val ssc: StreamingSparkContext = ...
    val cid = "ConsumerID"
    val topic = "sample-topic"
    val subExpression = "*"
    val accessKeyId = "kj7aY*******UYx6"
    val accessKeySecret = "AiNMAlxz*************1PxaPaL8t"
    val accessKeySecret = ""
    
    def func: Message => Array[Byte] = msg => msg.getBody
    
    val onsStream = OnsUtils.createStream(
     ssc,
     cId,
     topic,
     subExpression,
     accessKeyId,
     accessKeySecret,
     StorageLevel.MEMORY_AND_DISK_2,
     func)
    
     onsStream.foreachRDD(rdd => {
       ...
     })
    ssc

    StreamingContext object

    consumerId

    Name of a set of consumers

    topic

    Which topic to subscribe

    tags

    Which tag to subscribe

    accessKeyId

    Aliyun Access Key ID

    accessKeySecret

    Aliyun Access Key Secret

    storageLevel

    Storage level to use for storing the received objects. StorageLevel.MEMORY_AND_DISK_2 is recommended.

    func

    Extract information from ONS message

    returns

    Annotations
    @Experimental()
  10. def createStreams(ssc: StreamingContext, consumerIdTopicTags: Array[(String, String, String)], accessKeyId: String, accessKeySecret: String, storageLevel: StorageLevel, func: (Message) ⇒ Array[Byte]): DStream[Array[Byte]]

    Create an union input stream that pulls message from a Aliyun ONS stream.

    Create an union input stream that pulls message from a Aliyun ONS stream.

    val ssc: StreamingSparkContext = ...
    val consumerIdTopicTags = Array(("ConsumerID1", "sample-topic1", "*"))
    val accessKeyId = "kj7aY*******UYx6"
    val accessKeySecret = "AiNMAlxz*************1PxaPaL8t"
    val accessKeySecret = ""
    
    def func: Message => Array[Byte] = msg => msg.getBody
    
    val onsStream = OnsUtils.createStream(
     ssc,
    
     accessKeyId,
     accessKeySecret,
     StorageLevel.MEMORY_AND_DISK_2,
     func)
    
     onsStream.foreachRDD(rdd => {
       ...
     })
    ssc

    StreamingContext object

    consumerIdTopicTags

    Trituple(consumerId, topic, tag)

    accessKeyId

    Aliyun Access Key ID

    accessKeySecret

    Aliyun Access Key Secret

    storageLevel

    Storage level to use for storing the received objects. StorageLevel.MEMORY_AND_DISK_2 is recommended.

    func

    Extract information from ONS message

    returns

    Annotations
    @Experimental()
  11. final def eq(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef
  12. def equals(arg0: Any): Boolean

    Definition Classes
    AnyRef → Any
  13. def finalize(): Unit

    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  14. final def getClass(): Class[_]

    Definition Classes
    AnyRef → Any
  15. def hashCode(): Int

    Definition Classes
    AnyRef → Any
  16. final def isInstanceOf[T0]: Boolean

    Definition Classes
    Any
  17. final def ne(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef
  18. final def notify(): Unit

    Definition Classes
    AnyRef
  19. final def notifyAll(): Unit

    Definition Classes
    AnyRef
  20. final def synchronized[T0](arg0: ⇒ T0): T0

    Definition Classes
    AnyRef
  21. def toString(): String

    Definition Classes
    AnyRef → Any
  22. final def wait(): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  23. final def wait(arg0: Long, arg1: Int): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  24. final def wait(arg0: Long): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

Inherited from AnyRef

Inherited from Any

Ungrouped