q792602257/DataMate
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: fix the collection for nfs (#218)

Browse Source 
* fix: remove the datax-builder for the Backend Image

* fix: fix the collection for nfs
This commit is contained in:
hefanli
2025-12-31 15:56:01 +08:00
committed by GitHub
parent 01e1c6c7e9
commit 3a874fe699
5 changed files with 11 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
runtime/datax/nfsreader/src/main/java/com/modelengine/edatamate/plugin/reader/nfsreader/NfsReader.java
View File

@@ -69,12 +69,14 @@ public class NfsReader extends Reader {
private Configuration jobConfig;
private String mountPoint;
private Set<String> fileType;
private List<String> files;
@Override
public void init() {
this.jobConfig = super.getPluginJobConf();
this.mountPoint = this.jobConfig.getString("mountPoint");
this.fileType = new HashSet<>(this.jobConfig.getList("fileType", Collections.emptyList(), String.class));
this.files = this.jobConfig.getList("files", Collections.emptyList(), String.class);
}
@Override
@@ -83,6 +85,7 @@ public class NfsReader extends Reader {
List<String> files = stream.filter(Files::isRegularFile)
.filter(file -> fileType.isEmpty() || fileType.contains(getFileSuffix(file)))
.map(path -> path.getFileName().toString())
.filter(fileName -> this.files.isEmpty() || this.files.contains(fileName))
.collect(Collectors.toList());
files.forEach(filePath -> {
Record record = recordSender.createRecord();