You've already forked DataMate
feature：增加obs归集方式 (#90)
* feature：实现通过datax进行obs归集的插件 * feature：前端增加obs归集时的前缀参数
This commit is contained in:
Vincent
GitHub
88
runtime/datax/obswriter/pom.xml
Normal file
88
runtime/datax/obswriter/pom.xml
Normal file
@@ -0,0 +1,88 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.alibaba.datax</groupId>
|
||||
<artifactId>datax-all</artifactId>
|
||||
<version>0.0.1-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>obswriter</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.datax</groupId>
|
||||
<artifactId>datax-core</artifactId>
|
||||
<version>${datax-project-version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.datax</groupId>
|
||||
<artifactId>datax-common</artifactId>
|
||||
<version>${datax-project-version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-classic</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>s3</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>21</maven.compiler.source>
|
||||
<maven.compiler.target>21</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
|
||||
<build>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>src/main/java</directory>
|
||||
<includes>
|
||||
<include>**/*.properties</include>
|
||||
</includes>
|
||||
</resource>
|
||||
</resources>
|
||||
<plugins>
|
||||
<!-- compiler plugin -->
|
||||
<plugin>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<source>${jdk-version}</source>
|
||||
<target>${jdk-version}</target>
|
||||
<encoding>${project-sourceEncoding}</encoding>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<!-- assembly plugin -->
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<configuration>
|
||||
<descriptors>
|
||||
<descriptor>src/main/assembly/package.xml</descriptor>
|
||||
</descriptors>
|
||||
<finalName>datax</finalName>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>dwzip</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
35
runtime/datax/obswriter/src/main/assembly/package.xml
Normal file
35
runtime/datax/obswriter/src/main/assembly/package.xml
Normal file
@@ -0,0 +1,35 @@
|
||||
<assembly
|
||||
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||
<id></id>
|
||||
<formats>
|
||||
<format>dir</format>
|
||||
</formats>
|
||||
<includeBaseDirectory>false</includeBaseDirectory>
|
||||
<fileSets>
|
||||
<fileSet>
|
||||
<directory>src/main/resources</directory>
|
||||
<includes>
|
||||
<include>plugin.json</include>
|
||||
<include>plugin_job_template.json</include>
|
||||
</includes>
|
||||
<outputDirectory>plugin/writer/obswriter</outputDirectory>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>target/</directory>
|
||||
<includes>
|
||||
<include>obswriter-0.0.1-SNAPSHOT.jar</include>
|
||||
</includes>
|
||||
<outputDirectory>plugin/writer/obswriter</outputDirectory>
|
||||
</fileSet>
|
||||
</fileSets>
|
||||
|
||||
<dependencySets>
|
||||
<dependencySet>
|
||||
<useProjectArtifact>false</useProjectArtifact>
|
||||
<outputDirectory>plugin/writer/obswriter/libs</outputDirectory>
|
||||
<scope>runtime</scope>
|
||||
</dependencySet>
|
||||
</dependencySets>
|
||||
</assembly>
|
||||
@@ -0,0 +1,177 @@
|
||||
// java
|
||||
package com.datamate.plugin.writer.obswriter;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.alibaba.datax.common.element.Record;
|
||||
import com.alibaba.datax.common.element.StringColumn;
|
||||
import com.alibaba.datax.common.exception.CommonErrorCode;
|
||||
import com.alibaba.datax.common.exception.DataXException;
|
||||
import com.alibaba.datax.common.plugin.RecordReceiver;
|
||||
import com.alibaba.datax.common.spi.Writer;
|
||||
import com.alibaba.datax.common.util.Configuration;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
import software.amazon.awssdk.core.sync.ResponseTransformer;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.S3Configuration;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
||||
|
||||
public class ObsWriter extends Writer {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(ObsWriter.class);
|
||||
|
||||
public static class Job extends Writer.Job {
|
||||
private Configuration jobConfig = null;
|
||||
|
||||
@Override
|
||||
public void init() {
|
||||
this.jobConfig = super.getPluginJobConf();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void prepare() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Configuration> split(int adviceNumber) {
|
||||
return Collections.singletonList(this.jobConfig);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void post() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
}
|
||||
}
|
||||
|
||||
public static class Task extends Writer.Task {
|
||||
|
||||
private Configuration jobConfig;
|
||||
private Set<String> fileType;
|
||||
private String endpoint;
|
||||
private String accessKey;
|
||||
private String secretKey;
|
||||
private String bucket;
|
||||
private String destPath;
|
||||
private S3Client s3;
|
||||
|
||||
@Override
|
||||
public void init() {
|
||||
this.jobConfig = super.getPluginJobConf();
|
||||
this.fileType = new HashSet<>(this.jobConfig.getList("fileType", Collections.emptyList(), String.class));
|
||||
this.endpoint = this.jobConfig.getString("endpoint");
|
||||
this.accessKey = this.jobConfig.getString("accessKey");
|
||||
this.secretKey = this.jobConfig.getString("secretKey");
|
||||
this.bucket = this.jobConfig.getString("bucket");
|
||||
this.destPath = this.jobConfig.getString("destPath");
|
||||
this.s3 = getS3Client();
|
||||
}
|
||||
|
||||
private S3Client getS3Client() {
|
||||
try {
|
||||
AwsBasicCredentials creds = AwsBasicCredentials.create(accessKey, secretKey);
|
||||
S3Configuration serviceConfig = S3Configuration.builder()
|
||||
.pathStyleAccessEnabled(true)
|
||||
.build();
|
||||
return S3Client.builder()
|
||||
.endpointOverride(new URI(endpoint))
|
||||
.region(Region.of("us-east-1"))
|
||||
.serviceConfiguration(serviceConfig)
|
||||
.credentialsProvider(StaticCredentialsProvider.create(creds))
|
||||
.build();
|
||||
} catch (Exception e) {
|
||||
LOG.error("Error init s3 client: {}", this.endpoint, e);
|
||||
throw DataXException.asDataXException(CommonErrorCode.RUNTIME_ERROR, e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startWrite(RecordReceiver lineReceiver) {
|
||||
try {
|
||||
Record record;
|
||||
while ((record = lineReceiver.getFromReader()) != null) {
|
||||
String key = record.getColumn(0).asString();
|
||||
if (StringUtils.isBlank(key)) {
|
||||
continue;
|
||||
}
|
||||
copyFileFromObs(key);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOG.error("Error reading files from obs file system: {}", this.endpoint, e);
|
||||
throw DataXException.asDataXException(CommonErrorCode.RUNTIME_ERROR, e);
|
||||
}
|
||||
}
|
||||
|
||||
private void copyFileFromObs(String key) throws IOException {
|
||||
if (StringUtils.isBlank(endpoint) || StringUtils.isBlank(bucket)) {
|
||||
throw new IllegalArgumentException("endpoint and bucket must be provided");
|
||||
}
|
||||
try {
|
||||
// 确保目标目录存在
|
||||
Path targetDir = Paths.get(destPath);
|
||||
try {
|
||||
Files.createDirectories(targetDir);
|
||||
} catch (IOException e) {
|
||||
LOG.warn("Create dest dir {} failed: {}", targetDir, e.getMessage(), e);
|
||||
}
|
||||
|
||||
// 下载对象到本地目录，文件名取 key 最后一段
|
||||
String fileName = Paths.get(key).getFileName().toString();
|
||||
if (StringUtils.isBlank(fileName)) {
|
||||
// 防护：无法解析文件名则跳过下载
|
||||
LOG.warn("Skip object with empty file name for key {}", key);
|
||||
return;
|
||||
}
|
||||
Path target = targetDir.resolve(fileName);
|
||||
try {
|
||||
if (Files.exists(target)) {
|
||||
Files.delete(target);
|
||||
}
|
||||
GetObjectRequest getReq = GetObjectRequest.builder()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.build();
|
||||
s3.getObject(getReq, ResponseTransformer.toFile(target));
|
||||
LOG.info("Downloaded obs object {} to {}", key, target.toString());
|
||||
} catch (Exception ex) {
|
||||
LOG.warn("Failed to download object {}: {}", key, ex.getMessage(), ex);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOG.warn("Failed to build S3 client or download object {}: {}", key, e.getMessage(), e);
|
||||
// 保持原行为，对下载失败记录 warn，但不抛出新的运行时错误（外层会捕获）
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
if (s3 != null) {
|
||||
try {
|
||||
s3.close();
|
||||
} catch (Exception ignore) {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
6
runtime/datax/obswriter/src/main/resources/plugin.json
Normal file
6
runtime/datax/obswriter/src/main/resources/plugin.json
Normal file
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"name": "obswriter",
|
||||
"class": "com.datamate.plugin.writer.obswriter.ObsWriter",
|
||||
"description": "writer obs file to local",
|
||||
"developer": "datamate"
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "obswriter",
|
||||
"parameter": {
|
||||
"endpoint": "127.0.0.1",
|
||||
"bucket": "test",
|
||||
"accessKey": "ak-xxx",
|
||||
"secretKey": "sk-xxx",
|
||||
"prefix": "/test",
|
||||
"destPath": "/test"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user