StreamExecutionEnvironment内部的方法非常多,我们从整体功能上进行简单划分,主要包括:创建执行环境、设置执行环境、自定义配置、数据源输入、执行计算、计算设置以及各类查询操作等。
创建执行环境:
static LocalStreamEnvironment createLocalEnvironment()
static LocalStreamEnvironment createLocalEnvironment(Configuration configuration)
static LocalStreamEnvironment createLocalEnvironment(int parallelism)
static LocalStreamEnvironment createLocalEnvironment(int parallelism, Configuration configuration)
static StreamExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf)
static StreamExecutionEnvironment createRemoteEnvironment(String host, int port, Configuration clientConfig, String… jarFiles)
static StreamExecutionEnvironment createRemoteEnvironment(String host, int port, int parallelism, String… jarFiles)
static StreamExecutionEnvironment createRemoteEnvironment(String host, int port, String… jarFiles)
执行环境设置操作:
StreamExecutionEnvironment disableOperatorChaining()
StreamExecutionEnvironment enableCheckpointing() Deprecated.
StreamExecutionEnvironment enableCheckpointing(long interval)
StreamExecutionEnvironment enableCheckpointing(long interval, CheckpointingMode mode)
StreamExecutionEnvironment enableCheckpointing(long interval, CheckpointingMode mode, boolean force) Deprecated.
protected static void resetContextEnvironment()
StreamExecutionEnvironment setBufferTimeout(long timeoutMillis)
static void setDefaultLocalParallelism(int parallelism)
StreamExecutionEnvironment setDefaultSavepointDirectory(Path savepointDirectory)
StreamExecutionEnvironment setDefaultSavepointDirectory(String savepointDirectory)
StreamExecutionEnvironment setDefaultSavepointDirectory(URI savepointDirectory)
StreamExecutionEnvironment setMaxParallelism(int maxParallelism)
void setNumberOfExecutionRetries(int numberOfExecutionRetries) Deprecated.
StreamExecutionEnvironment setParallelism(int parallelism)
void setRestartStrategy(RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration)
StreamExecutionEnvironment setRuntimeMode(RuntimeExecutionMode executionMode)
StreamExecutionEnvironment setStateBackend(StateBackend backend)
void setStreamTimeCharacteristic(TimeCharacteristic characteristic) Deprecated.
配置:
void configure(ReadableConfig configuration)
void configure(ReadableConfig configuration, ClassLoader classLoader)
protected static void initializeContextEnvironment(StreamExecutionEnvironmentFactory ctx)
数据输入:
DataStreamSource addSource(SourceFunction function)
DataStreamSource addSource(SourceFunction function, String sourceName)
DataStreamSource addSource(SourceFunction function, String sourceName, TypeInformation typeInfo)
DataStreamSource addSource(SourceFunction function, TypeInformation typeInfo)
DataStreamSource createInput(InputFormat inputFormat)
DataStreamSource createInput(InputFormat inputFormat, TypeInformation typeInfo)
DataStreamSource fromCollection(Collection data)
DataStreamSource fromCollection(Collection data, TypeInformation typeInfo)
DataStreamSource fromCollection(Iterator data, Class type)
DataStreamSource fromCollection(Iterator data, TypeInformation typeInfo)
DataStreamSource fromElements(Class type, OUT… data)
DataStreamSource fromElements(OUT… data)
DataStreamSource fromParallelCollection(SplittableIterator iterator, Class type)
DataStreamSource fromParallelCollection(SplittableIterator iterator, TypeInformation typeInfo)
DataStreamSource fromSequence(long from, long to)
DataStreamSource fromSource(Source source, WatermarkStrategy timestampsAndWatermarks, String sourceName)
DataStreamSource fromSource(Source source, WatermarkStrategy timestampsAndWatermarks, String sourceName, TypeInformation typeInfo)
DataStreamSource generateSequence(long from, long to)
DataStreamSource readFile(FileInputFormat inputFormat, String filePath)
DataStreamSource readFile(FileInputFormat inputFormat, String filePath, FileProcessingMode watchType, long interval)
DataStreamSource readFile(FileInputFormat inputFormat, String filePath, FileProcessingMode watchType, long interval, FilePathFilter filter)
DataStreamSource readFile(FileInputFormat inputFormat, String filePath, FileProcessingMode watchType, long interval, TypeInformation typeInformation)
DataStream readFileStream(String filePath, long intervalMillis, FileMonitoringFunction.WatchType watchType)
DataStreamSource readTextFile(String filePath)
DataStreamSource readTextFile(String filePath, String charsetName)
DataStreamSource socketTextStream(String hostname, int port)
DataStreamSource socketTextStream(String hostname, int port, char delimiter)
DataStreamSource socketTextStream(String hostname, int port, char delimiter, long maxRetry) Deprecated.
DataStreamSource socketTextStream(String hostname, int port, String delimiter)
DataStreamSource socketTextStream(String hostname, int port, String delimiter, long maxRetry)
执行计算:
JobExecutionResult execute()
JobExecutionResult execute(StreamGraph streamGraph)
JobExecutionResult execute(String jobName)
JobClient executeAsync()
JobClient executeAsync(StreamGraph streamGraph)
JobClient executeAsync(String jobName)
执行设置:
void registerCachedFile(String filePath, String name)
void registerCachedFile(String filePath, String name, boolean executable)
void registerJobListener(JobListener jobListener)
StreamExecutionEnvironment registerSlotSharingGroup(SlotSharingGroup slotSharingGroup)
void registerType(Class type)
void registerTypeWithKryoSerializer(Class type, Class serializerClass)
void registerTypeWithKryoSerializer(Class type, T serializer)
void addOperator(Transformation transformation)
F clean(F f)
void clearJobListeners()
StreamGraph generateStreamGraph(List> transformations)
查询相关操作:
long getBufferTimeout()
List> getCachedFiles()
CheckpointConfig getCheckpointConfig()
CheckpointingMode getCheckpointingMode()
long getCheckpointInterval()
ExecutionConfig getConfig()
ReadableConfig getConfiguration()
static int getDefaultLocalParallelism()
Path getDefaultSavepointDirectory()
static StreamExecutionEnvironment getExecutionEnvironment()
static StreamExecutionEnvironment getExecutionEnvironment(Configuration configuration)
String getExecutionPlan()
List getJobListeners()
int getMaxParallelism()
int getNumberOfExecutionRetries()
int getParallelism()
StateBackend getStateBackend()
StreamGraph getStreamGraph()
StreamGraph getStreamGraph(boolean clearTransformations)
TimeCharacteristic getStreamTimeCharacteristic()
protected ClassLoader getUserClassloader()
boolean isChainingEnabled()
boolean isForceCheckpointing()
boolean isForceUnalignedCheckpoints()
boolean isUnalignedCheckpointsEnabled()
其它操作:
void addDefaultKryoSerializer(Class type, Class> serializerClass)
void addDefaultKryoSerializer(Class type, T serializer)