ETL采集器-开发者模式

ETL采集器-开发者模式

ETL采集器一共有4层分别为：采集层、任务拆分层、清洗层、存储层。除了

采集层开发者API

实现ICollectData接口,存储本地文件

public interface ICollectData {
//执行采集过程
public IData<String, String>  execute(String id,CollectorJobRunState collectorJobRunState) throws Exception ;
//采集同步
public IData<String, String> synchronous(String id,CollectorJobRunState collectorJobRunState) throws Exception;
//自动采集
public IData<String, String> collectAuto(String id,CollectorJobRunState collectorJobRunState) throws Exception;
//获取采集参数
public Map<String, String> getParameter();
//初始化采集参数
public void initCollectParameter(String id);
//设置参数
public void setCollectParameter(String id,IData<String,String> data);
}

将实现类加入spring容器中

</bean>

清洗开发者API

实现CleanDataMethod清洗接口

public interface CleanDataMethod {

/**
*
* @Title: getForMatData
* @Description: TODO(清洗数据方法)
* @param valueThis 原始日志
* @param parameters 参数
* @param elementXml xml属性
* @param data 保存数据
* @param key 字段
* @param id id
* @return
* @throws CollectorExceptionLog
* @return ForMatReturn    返回类型
* @throws
*/
public IData<String,Object> executeCleanData(String valueThis,String parameters,String elementXml, IData<String,Object> data,String key,String id)throws CollectorExceptionLog;
}

将实现类加入spring容器中

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans
classpath:spring-beans.xsd">

<bean id="mappingArray"
class="com.venustech.collector.service.fromat.method.mapping.MappingArray" />
<bean id="mappingID"
class="com.venustech.collector.service.fromat.method.mapping.MappingID" />
<bean id="mappingValueArray"
class="com.venustech.collector.service.fromat.method.mapping.MappingValueArray" />
<bean id="mappingValueID"
class="com.venustech.collector.service.fromat.method.mapping.MappingValueID" />
<bean id="dateUnix"
class="com.venustech.collector.service.fromat.method.dateunix.DateUnix" />
<bean id="dateUnixArray"
class="com.venustech.collector.service.fromat.method.dateunix.DateUnixArray" />
<bean id="dateUnixArrayTime"
class="com.venustech.collector.service.fromat.method.dateunix.DateUnixArrayTime" />
<bean id="dateUnixID"
class="com.venustech.collector.service.fromat.method.dateunix.DateUnixID" />
<bean id="dateUnixIDTime"
class="com.venustech.collector.service.fromat.method.dateunix.DateUnixIDTime" />
<bean id="iPIntArray"
class="com.venustech.collector.service.fromat.method.iptoint.IPIntArray" />
<bean id="iPIntID"
class="com.venustech.collector.service.fromat.method.iptoint.IPIntID" />
<bean id="regexpArray"
class="com.venustech.collector.service.fromat.method.regexp.RegexpArray" />
<bean id="regexpID"
class="com.venustech.collector.service.fromat.method.regexp.RegexpID" />
<bean id="regexpThis"
class="com.venustech.collector.service.fromat.method.regexp.RegexpThis" />
<bean id="splitArray"
class="com.venustech.collector.service.fromat.method.split.SplitArray" />
<bean id="splitID"
class="com.venustech.collector.service.fromat.method.split.SplitID" />
<bean id="splitThis"
class="com.venustech.collector.service.fromat.method.split.SplitThis" />
<bean id="textArray"
class="com.venustech.collector.service.fromat.method.text.TextArray" />
<bean id="textID"
class="com.venustech.collector.service.fromat.method.text.TextID" />
<bean id="info"
class="com.venustech.collector.service.fromat.method.info.Info" />
<bean id="subStringArray"
class="com.venustech.collector.service.fromat.method.substring.SubStringArray" />
<bean id="filterRegexpArray"
class="com.venustech.collector.service.fromat.method.regexp.FilterRegexpArray" />
<bean id="filterRegexpID"
class="com.venustech.collector.service.fromat.method.regexp.FilterRegexpID" />
<bean id="filterRegexpThis"
class="com.venustech.collector.service.fromat.method.regexp.FilterRegexpThis" />
<bean id="splitKeyValueID"
class="com.venustech.collector.service.fromat.method.split.SplitKeyValueID" />

<bean id="subStringID"
class="com.venustech.collector.service.fromat.method.substring.SubStringID" />
<bean id="completionData"
class="com.venustech.collector.service.fromat.method.completion.CompletionData" />
<bean id="completionDataDefault"
class="com.venustech.collector.service.fromat.method.completion.CompletionDataDefault" />
<bean id="subIndexStringArray"
class="com.venustech.collector.service.fromat.method.substring.SubIndexStringArray" />
<bean id="subIndexStringId"
class="com.venustech.collector.service.fromat.method.substring.SubIndexStringId" />
<bean id="urlToID" class="com.venustech.collector.service.fromat.method.urltoip.UrlToID" />
<bean id="urlToIpArray"
class="com.venustech.collector.service.fromat.method.urltoip.UrlToIpArray" />
<bean id="mappingKey"
class="com.venustech.collector.service.fromat.method.mapping.MappingKey" />
<bean id="mappingValueKey"
class="com.venustech.collector.service.fromat.method.mapping.MappingValueKey" />

<bean id="toDateArray"
class="com.venustech.collector.service.fromat.method.todate.ToDateArray" />

<bean id="toDateIDTime"
class="com.venustech.collector.service.fromat.method.todate.ToDateIDTime" />

<bean id="mainformat" class="com.venustech.collector.service.fromat.MainForMat">
<property name="method">
<map>
<entry key="mappingValueKey" value-ref="mappingValueKey" />
<entry key="mappingKey" value-ref="mappingKey" />
<entry key="mappingArray" value-ref="mappingArray" />
<entry key="mappingID" value-ref="mappingID" />
<entry key="mappingValueArray" value-ref="mappingValueArray" />
<entry key="mappingValueID" value-ref="mappingValueID" />
<entry key="dateUnix" value-ref="dateUnix" />
<entry key="dateUnixArray" value-ref="dateUnixArray" />
<entry key="dateUnixArrayTime" value-ref="dateUnixArrayTime" />
<entry key="dateUnixID" value-ref="dateUnixID" />
<entry key="dateUnixIDTime" value-ref="dateUnixIDTime" />
<entry key="iPIntArray" value-ref="iPIntArray" />
<entry key="iPIntID" value-ref="iPIntID" />
<entry key="regexpArray" value-ref="regexpArray" />
<entry key="regexpID" value-ref="regexpID" />
<entry key="regexpThis" value-ref="regexpThis" />
<entry key="splitArray" value-ref="splitArray" />
<entry key="splitID" value-ref="splitID" />
<entry key="splitThis" value-ref="splitThis" />
<entry key="textArray" value-ref="textArray" />
<entry key="textID" value-ref="textID" />
<entry key="subStringID" value-ref="subStringID" />
<entry key="subStringArray" value-ref="subStringArray" />
<entry key="completionData" value-ref="completionData" />
<entry key="subIndexStringArray" value-ref="subIndexStringArray" />
<entry key="subIndexStringId" value-ref="subIndexStringId" />
<entry key="completionDataDefault" value-ref="completionDataDefault" />
<entry key="urlToID" value-ref="urlToID" />
<entry key="urlToIpArray" value-ref="urlToIpArray" />
<entry key="filterRegexpArray" value-ref="filterRegexpArray" />
<entry key="filterRegexpID" value-ref="filterRegexpID" />
<entry key="filterRegexpThis" value-ref="filterRegexpThis" />
<entry key="splitKeyValueID" value-ref="splitKeyValueID" />
<!--映射 -->
<entry key="mappingArray" value-ref="mappingArray" />
<entry key="mappingID" value-ref="mappingID" />
<entry key="mappingValueArray" value-ref="mappingValueArray" />
<entry key="mappingValueID" value-ref="mappingValueID" />
<entry key="mappingKey" value-ref="mappingKey" />
<entry key="mappingValueKey" value-ref="mappingValueKey" />
<entry key="info" value-ref="info" />
<!--補全 -->
<entry key="completionData" value-ref="completionData" />
<entry key="completionDataDefault" value-ref="completionDataDefault" />
<!--data转换-->
<entry key="toDateIDTime" value-ref="toDateIDTime" />
<entry key="toDateArray" value-ref="toDateArray" />
</map>
</property>
</bean>

</beans>

存储层开发者API

实现SaveCollector接口

public interface SaveCollector{
public String save(String id, String[] tableName,
IDataset<IData<String, Object>> list,FileAttribute fileAttribute) throws Exception;

}

将实现类加入spring容器中