You've already forked DataMate
hhhhsc701
GitHub
2
Makefile
2
Makefile
@@ -209,7 +209,7 @@ endif
|
||||
.PHONY: uninstall
|
||||
uninstall:
|
||||
ifeq ($(origin INSTALLER), undefined)
|
||||
$(call prompt-uninstaller,milvus-$$INSTALLER-uninstall label-studio-$$INSTALLER-uninstall datamate-$$INSTALLER-uninstall)
|
||||
$(call prompt-uninstaller,milvus-$$INSTALLER-uninstall datamate-$$INSTALLER-uninstall)
|
||||
else
|
||||
@echo "Delete volumes? (This will remove all data)"; \
|
||||
echo "1. Yes - Delete volumes"; \
|
||||
|
||||
@@ -248,4 +248,8 @@ public class CleaningTaskService {
|
||||
public void stopTask(String taskId) {
|
||||
taskScheduler.stopTask(taskId);
|
||||
}
|
||||
|
||||
public List<OperatorInstanceDto> getInstanceByTemplateId(String templateId) {
|
||||
return operatorInstanceRepo.findInstanceByInstanceId(templateId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package com.datamate.cleaning.application;
|
||||
|
||||
import com.datamate.cleaning.domain.repository.CleaningTemplateRepository;
|
||||
import com.datamate.cleaning.domain.repository.OperatorInstanceRepository;
|
||||
import com.datamate.cleaning.infrastructure.validator.CleanTaskValidator;
|
||||
import com.datamate.cleaning.interfaces.dto.*;
|
||||
import com.datamate.cleaning.domain.model.entity.TemplateWithInstance;
|
||||
import com.datamate.operator.domain.repository.OperatorViewRepository;
|
||||
@@ -28,6 +29,8 @@ public class CleaningTemplateService {
|
||||
|
||||
private final OperatorViewRepository operatorViewRepo;
|
||||
|
||||
private final CleanTaskValidator cleanTaskValidator;
|
||||
|
||||
public List<CleaningTemplateDto> getTemplates(String keywords) {
|
||||
List<OperatorDto> allOperators =
|
||||
operatorViewRepo.findOperatorsByCriteria(null, null, null, null, null);
|
||||
@@ -59,6 +62,7 @@ public class CleaningTemplateService {
|
||||
|
||||
@Transactional
|
||||
public CleaningTemplateDto createTemplate(CreateCleaningTemplateRequest request) {
|
||||
cleanTaskValidator.checkInputAndOutput(request.getInstance());
|
||||
CleaningTemplateDto template = new CleaningTemplateDto();
|
||||
String templateId = UUID.randomUUID().toString();
|
||||
template.setId(templateId);
|
||||
|
||||
@@ -13,4 +13,6 @@ public interface OperatorInstanceRepository extends IRepository<OperatorInstance
|
||||
void deleteByInstanceId(String instanceId);
|
||||
|
||||
List<OperatorDto> findOperatorByInstanceId(String instanceId);
|
||||
|
||||
List<OperatorInstanceDto> findInstanceByInstanceId(String instanceId);
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import com.datamate.common.infrastructure.exception.SystemErrorCode;
|
||||
import com.datamate.operator.domain.model.OperatorView;
|
||||
import com.datamate.operator.interfaces.dto.OperatorDto;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.mapstruct.Mapper;
|
||||
import org.mapstruct.Mapping;
|
||||
@@ -23,20 +24,39 @@ import java.util.Map;
|
||||
public interface OperatorInstanceConverter {
|
||||
OperatorInstanceConverter INSTANCE = Mappers.getMapper(OperatorInstanceConverter.class);
|
||||
|
||||
ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
|
||||
@Mapping(target = "settingsOverride", source = "overrides", qualifiedByName = "mapToString")
|
||||
@Mapping(target = "operatorId", source = "id")
|
||||
OperatorInstance fromDtoToEntity(OperatorInstanceDto instance);
|
||||
|
||||
@Mapping(target = "overrides", source = "settingsOverride", qualifiedByName = "stringToMap")
|
||||
@Mapping(target = "id", source = "operatorId")
|
||||
OperatorInstanceDto fromEntityToDto(OperatorInstance instance);
|
||||
|
||||
List<OperatorInstanceDto> fromEntityToDtoList(List<OperatorInstance> instance);
|
||||
|
||||
@Named("mapToString")
|
||||
static String mapToString(Map<String, Object> objects) {
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
try {
|
||||
return objectMapper.writeValueAsString(objects);
|
||||
return OBJECT_MAPPER.writeValueAsString(objects);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw BusinessException.of(SystemErrorCode.UNKNOWN_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
@Named("stringToMap")
|
||||
static Map<String, Object> stringToMap(String json) {
|
||||
if (json == null) {
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
try {
|
||||
return OBJECT_MAPPER.readValue(json, new TypeReference<>() {});
|
||||
} catch (JsonProcessingException e) {
|
||||
throw BusinessException.of(SystemErrorCode.UNKNOWN_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
@Mapping(target = "categories", source = "categories", qualifiedByName = "stringToList")
|
||||
OperatorDto fromEntityToDto(OperatorView operator);
|
||||
|
||||
|
||||
@@ -42,4 +42,12 @@ public class OperatorInstanceRepositoryImpl extends CrudRepository<OperatorInsta
|
||||
public List<OperatorDto> findOperatorByInstanceId(String instanceId) {
|
||||
return OperatorInstanceConverter.INSTANCE.fromEntityToDto(mapper.findOperatorByInstanceId(instanceId));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<OperatorInstanceDto> findInstanceByInstanceId(String instanceId) {
|
||||
LambdaQueryWrapper<OperatorInstance> lambdaWrapper = new LambdaQueryWrapper<>();
|
||||
lambdaWrapper.eq(OperatorInstance::getInstanceId, instanceId)
|
||||
.orderByAsc(OperatorInstance::getOpIndex);
|
||||
return OperatorInstanceConverter.INSTANCE.fromEntityToDtoList(mapper.selectList(lambdaWrapper));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,6 +27,8 @@ public class CreateCleaningTaskRequest {
|
||||
|
||||
private String destDatasetType;
|
||||
|
||||
private String templateId;
|
||||
|
||||
private List<OperatorInstanceDto> instance = new ArrayList<>();
|
||||
}
|
||||
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
package com.datamate.cleaning.interfaces.rest;
|
||||
|
||||
import com.datamate.cleaning.application.CleaningTaskService;
|
||||
import com.datamate.cleaning.interfaces.dto.CleaningResultDto;
|
||||
import com.datamate.cleaning.interfaces.dto.CleaningTaskDto;
|
||||
import com.datamate.cleaning.interfaces.dto.CleaningTaskLog;
|
||||
import com.datamate.cleaning.interfaces.dto.CreateCleaningTaskRequest;
|
||||
import com.datamate.cleaning.interfaces.dto.*;
|
||||
import com.datamate.common.interfaces.PagedResponse;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.List;
|
||||
@@ -31,6 +29,9 @@ public class CleaningTaskController {
|
||||
|
||||
@PostMapping
|
||||
public CleaningTaskDto cleaningTasksPost(@RequestBody CreateCleaningTaskRequest request) {
|
||||
if (request.getInstance().isEmpty() && StringUtils.isNotBlank(request.getTemplateId())) {
|
||||
request.setInstance(cleaningTaskService.getInstanceByTemplateId(request.getTemplateId()));
|
||||
}
|
||||
return cleaningTaskService.createTask(request);
|
||||
}
|
||||
|
||||
@@ -57,6 +58,13 @@ public class CleaningTaskController {
|
||||
return taskId;
|
||||
}
|
||||
|
||||
@DeleteMapping
|
||||
public void cleaningTasksDelete(@RequestParam List<String> taskIds) {
|
||||
for (String taskId : taskIds) {
|
||||
cleaningTaskService.deleteTask(taskId);
|
||||
}
|
||||
}
|
||||
|
||||
@GetMapping("/{taskId}/result")
|
||||
public List<CleaningResultDto> cleaningTasksTaskIdGetResult(@PathVariable("taskId") String taskId) {
|
||||
return cleaningTaskService.getTaskResults(taskId);
|
||||
|
||||
@@ -102,261 +102,261 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
{{- $replicaCount := int .Values.replicaCount }}
|
||||
{{- $peerPort := int .Values.containerPorts.peer }}
|
||||
{{- $etcdFullname := include "common.names.fullname" . }}
|
||||
{{- $releaseNamespace := .Release.Namespace }}
|
||||
{{- $etcdHeadlessServiceName := (printf "%s-%s" $etcdFullname "headless" | trunc 63 | trimSuffix "-") }}
|
||||
{{- $clusterDomain := .Values.clusterDomain }}
|
||||
{{- $etcdPeerProtocol := include "etcd.peerProtocol" . }}
|
||||
{{- $etcdClientProtocol := include "etcd.clientProtocol" . }}
|
||||
- name: etcd
|
||||
image: {{ include "etcd.image" . }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
|
||||
{{- if .Values.containerSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- $peerPort := int .Values.containerPorts.peer }}
|
||||
{{- $etcdFullname := include "common.names.fullname" . }}
|
||||
{{- $releaseNamespace := .Release.Namespace }}
|
||||
{{- $etcdHeadlessServiceName := (printf "%s-%s" $etcdFullname "headless" | trunc 63 | trimSuffix "-") }}
|
||||
{{- $clusterDomain := .Values.clusterDomain }}
|
||||
{{- $etcdPeerProtocol := include "etcd.peerProtocol" . }}
|
||||
{{- $etcdClientProtocol := include "etcd.clientProtocol" . }}
|
||||
- name: etcd
|
||||
image: {{ include "etcd.image" . }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
|
||||
{{- if .Values.containerSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- if .Values.diagnosticMode.enabled }}
|
||||
command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
|
||||
{{- else if .Values.command }}
|
||||
command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- if .Values.diagnosticMode.enabled }}
|
||||
args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
|
||||
{{- else if .Values.args }}
|
||||
args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 14 }}
|
||||
{{- end }}
|
||||
env:
|
||||
- name: BITNAMI_DEBUG
|
||||
value: {{ ternary "true" "false" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }}
|
||||
- name: MY_POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.podIP
|
||||
- name: MY_POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: MY_STS_NAME
|
||||
value: {{ include "common.names.fullname" . | quote }}
|
||||
- name: ETCDCTL_API
|
||||
value: "3"
|
||||
- name: ETCD_ON_K8S
|
||||
value: "yes"
|
||||
- name: ETCD_START_FROM_SNAPSHOT
|
||||
value: {{ ternary "yes" "no" .Values.startFromSnapshot.enabled | quote }}
|
||||
- name: ETCD_DISASTER_RECOVERY
|
||||
value: {{ ternary "yes" "no" .Values.disasterRecovery.enabled | quote }}
|
||||
- name: ETCD_NAME
|
||||
value: "$(MY_POD_NAME)"
|
||||
- name: ETCD_DATA_DIR
|
||||
value: "/bitnami/etcd/data"
|
||||
- name: ETCD_LOG_LEVEL
|
||||
value: {{ ternary "debug" .Values.logLevel .Values.image.debug | quote }}
|
||||
- name: ALLOW_NONE_AUTHENTICATION
|
||||
value: {{ ternary "yes" "no" (and (not (or .Values.auth.rbac.create .Values.auth.rbac.enabled)) .Values.auth.rbac.allowNoneAuthentication) | quote }}
|
||||
{{- if or .Values.auth.rbac.create .Values.auth.rbac.enabled }}
|
||||
- name: ETCD_ROOT_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "etcd.secretName" . }}
|
||||
key: {{ include "etcd.secretPasswordKey" . }}
|
||||
{{- end }}
|
||||
{{- if .Values.diagnosticMode.enabled }}
|
||||
command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
|
||||
{{- else if .Values.command }}
|
||||
command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 14 }}
|
||||
{{- if .Values.auth.token.enabled }}
|
||||
- name: ETCD_AUTH_TOKEN
|
||||
{{- if eq .Values.auth.token.type "jwt" }}
|
||||
value: {{ printf "jwt,priv-key=/opt/bitnami/etcd/certs/token/%s,sign-method=%s,ttl=%s" .Values.auth.token.privateKey.filename .Values.auth.token.signMethod .Values.auth.token.ttl | quote }}
|
||||
{{- else if eq .Values.auth.token.type "simple" }}
|
||||
value: "simple"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.diagnosticMode.enabled }}
|
||||
args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
|
||||
{{- else if .Values.args }}
|
||||
args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 14 }}
|
||||
# 警告：以下 Env Vars 依赖 StatefulSet 命名 (headless service 和 ordered names)
|
||||
# 在 Deployment 中如果不修改副本数为 1，这些配置会导致集群发现失败。
|
||||
- name: ETCD_ADVERTISE_CLIENT_URLS
|
||||
value: "{{ $etcdClientProtocol }}://$(MY_POD_NAME).{{ $etcdHeadlessServiceName }}.{{ .Release.Namespace }}.svc.{{ $clusterDomain }}:{{ .Values.containerPorts.client }},{{ $etcdClientProtocol }}://{{ $etcdFullname }}.{{ .Release.Namespace }}.svc.{{ $clusterDomain }}:{{ coalesce .Values.service.ports.client .Values.service.port }}"
|
||||
- name: ETCD_LISTEN_CLIENT_URLS
|
||||
value: "{{ $etcdClientProtocol }}://0.0.0.0:{{ .Values.containerPorts.client }}"
|
||||
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
|
||||
value: "{{ $etcdPeerProtocol }}://$(MY_POD_NAME).{{ $etcdHeadlessServiceName }}.{{ .Release.Namespace }}.svc.{{ $clusterDomain }}:{{ .Values.containerPorts.peer }}"
|
||||
- name: ETCD_LISTEN_PEER_URLS
|
||||
value: "{{ $etcdPeerProtocol }}://0.0.0.0:{{ .Values.containerPorts.peer }}"
|
||||
{{- if .Values.autoCompactionMode }}
|
||||
- name: ETCD_AUTO_COMPACTION_MODE
|
||||
value: {{ .Values.autoCompactionMode | quote }}
|
||||
{{- end }}
|
||||
env:
|
||||
- name: BITNAMI_DEBUG
|
||||
value: {{ ternary "true" "false" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }}
|
||||
- name: MY_POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.podIP
|
||||
- name: MY_POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: MY_STS_NAME
|
||||
value: {{ include "common.names.fullname" . | quote }}
|
||||
- name: ETCDCTL_API
|
||||
value: "3"
|
||||
- name: ETCD_ON_K8S
|
||||
value: "yes"
|
||||
- name: ETCD_START_FROM_SNAPSHOT
|
||||
value: {{ ternary "yes" "no" .Values.startFromSnapshot.enabled | quote }}
|
||||
- name: ETCD_DISASTER_RECOVERY
|
||||
value: {{ ternary "yes" "no" .Values.disasterRecovery.enabled | quote }}
|
||||
- name: ETCD_NAME
|
||||
value: "$(MY_POD_NAME)"
|
||||
- name: ETCD_DATA_DIR
|
||||
value: "/bitnami/etcd/data"
|
||||
- name: ETCD_LOG_LEVEL
|
||||
value: {{ ternary "debug" .Values.logLevel .Values.image.debug | quote }}
|
||||
- name: ALLOW_NONE_AUTHENTICATION
|
||||
value: {{ ternary "yes" "no" (and (not (or .Values.auth.rbac.create .Values.auth.rbac.enabled)) .Values.auth.rbac.allowNoneAuthentication) | quote }}
|
||||
{{- if or .Values.auth.rbac.create .Values.auth.rbac.enabled }}
|
||||
- name: ETCD_ROOT_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "etcd.secretName" . }}
|
||||
key: {{ include "etcd.secretPasswordKey" . }}
|
||||
{{- end }}
|
||||
{{- if .Values.auth.token.enabled }}
|
||||
- name: ETCD_AUTH_TOKEN
|
||||
{{- if eq .Values.auth.token.type "jwt" }}
|
||||
value: {{ printf "jwt,priv-key=/opt/bitnami/etcd/certs/token/%s,sign-method=%s,ttl=%s" .Values.auth.token.privateKey.filename .Values.auth.token.signMethod .Values.auth.token.ttl | quote }}
|
||||
{{- else if eq .Values.auth.token.type "simple" }}
|
||||
value: "simple"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
# 警告：以下 Env Vars 依赖 StatefulSet 命名 (headless service 和 ordered names)
|
||||
# 在 Deployment 中如果不修改副本数为 1，这些配置会导致集群发现失败。
|
||||
- name: ETCD_ADVERTISE_CLIENT_URLS
|
||||
value: "{{ $etcdClientProtocol }}://$(MY_POD_NAME).{{ $etcdHeadlessServiceName }}.{{ .Release.Namespace }}.svc.{{ $clusterDomain }}:{{ .Values.containerPorts.client }},{{ $etcdClientProtocol }}://{{ $etcdFullname }}.{{ .Release.Namespace }}.svc.{{ $clusterDomain }}:{{ coalesce .Values.service.ports.client .Values.service.port }}"
|
||||
- name: ETCD_LISTEN_CLIENT_URLS
|
||||
value: "{{ $etcdClientProtocol }}://0.0.0.0:{{ .Values.containerPorts.client }}"
|
||||
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
|
||||
value: "{{ $etcdPeerProtocol }}://$(MY_POD_NAME).{{ $etcdHeadlessServiceName }}.{{ .Release.Namespace }}.svc.{{ $clusterDomain }}:{{ .Values.containerPorts.peer }}"
|
||||
- name: ETCD_LISTEN_PEER_URLS
|
||||
value: "{{ $etcdPeerProtocol }}://0.0.0.0:{{ .Values.containerPorts.peer }}"
|
||||
{{- if .Values.autoCompactionMode }}
|
||||
- name: ETCD_AUTO_COMPACTION_MODE
|
||||
value: {{ .Values.autoCompactionMode | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.autoCompactionRetention }}
|
||||
- name: ETCD_AUTO_COMPACTION_RETENTION
|
||||
value: {{ .Values.autoCompactionRetention | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.maxProcs }}
|
||||
- name: GOMAXPROCS
|
||||
value: {{ .Values.maxProcs }}
|
||||
{{- end }}
|
||||
{{- if gt $replicaCount 1 }}
|
||||
- name: ETCD_INITIAL_CLUSTER_TOKEN
|
||||
value: "etcd-cluster-k8s"
|
||||
- name: ETCD_INITIAL_CLUSTER_STATE
|
||||
value: {{ default (ternary "new" "existing" .Release.IsInstall) .Values.initialClusterState | quote }}
|
||||
{{- $initialCluster := list }}
|
||||
{{- range $e, $i := until $replicaCount }}
|
||||
{{- $initialCluster = append $initialCluster (printf "%s-%d=%s://%s-%d.%s.%s.svc.%s:%d" $etcdFullname $i $etcdPeerProtocol $etcdFullname $i $etcdHeadlessServiceName $releaseNamespace $clusterDomain $peerPort) }}
|
||||
{{- end }}
|
||||
- name: ETCD_INITIAL_CLUSTER
|
||||
value: {{ join "," $initialCluster | quote }}
|
||||
{{- end }}
|
||||
- name: ETCD_CLUSTER_DOMAIN
|
||||
value: {{ printf "%s.%s.svc.%s" $etcdHeadlessServiceName $releaseNamespace $clusterDomain | quote }}
|
||||
{{- if and .Values.auth.client.secureTransport .Values.auth.client.useAutoTLS }}
|
||||
- name: ETCD_AUTO_TLS
|
||||
value: "true"
|
||||
{{- else if .Values.auth.client.secureTransport }}
|
||||
- name: ETCD_CERT_FILE
|
||||
value: "/opt/bitnami/etcd/certs/client/{{ .Values.auth.client.certFilename }}"
|
||||
- name: ETCD_KEY_FILE
|
||||
value: "/opt/bitnami/etcd/certs/client/{{ .Values.auth.client.certKeyFilename }}"
|
||||
{{- if .Values.auth.client.enableAuthentication }}
|
||||
- name: ETCD_CLIENT_CERT_AUTH
|
||||
value: "true"
|
||||
- name: ETCD_TRUSTED_CA_FILE
|
||||
value: "/opt/bitnami/etcd/certs/client/{{ .Values.auth.client.caFilename | default "ca.crt" }}"
|
||||
{{- else if .Values.auth.client.caFilename }}
|
||||
- name: ETCD_TRUSTED_CA_FILE
|
||||
value: "/opt/bitnami/etcd/certs/client/{{ .Values.auth.client.caFilename | default "ca.crt" }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if and .Values.auth.peer.secureTransport .Values.auth.peer.useAutoTLS }}
|
||||
- name: ETCD_PEER_AUTO_TLS
|
||||
value: "true"
|
||||
{{- else if .Values.auth.peer.secureTransport }}
|
||||
- name: ETCD_PEER_CERT_FILE
|
||||
value: "/opt/bitnami/etcd/certs/peer/{{ .Values.auth.peer.certFilename }}"
|
||||
- name: ETCD_PEER_KEY_FILE
|
||||
value: "/opt/bitnami/etcd/certs/peer/{{ .Values.auth.peer.certKeyFilename }}"
|
||||
{{- if .Values.auth.peer.enableAuthentication }}
|
||||
- name: ETCD_PEER_CLIENT_CERT_AUTH
|
||||
value: "true"
|
||||
- name: ETCD_PEER_TRUSTED_CA_FILE
|
||||
value: "/opt/bitnami/etcd/certs/peer/{{ .Values.auth.peer.caFilename | default "ca.crt" }}"
|
||||
{{- else if .Values.auth.peer.caFilename }}
|
||||
- name: ETCD_PEER_TRUSTED_CA_FILE
|
||||
value: "/opt/bitnami/etcd/certs/peer/{{ .Values.auth.peer.caFilename | default "ca.crt" }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.startFromSnapshot.enabled }}
|
||||
- name: ETCD_INIT_SNAPSHOT_FILENAME
|
||||
value: {{ .Values.startFromSnapshot.snapshotFilename | quote }}
|
||||
- name: ETCD_INIT_SNAPSHOTS_DIR
|
||||
value: {{ ternary "/snapshots" "/init-snapshot" (and .Values.disasterRecovery.enabled (not .Values.disasterRecovery.pvc.existingClaim)) | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.extraEnvVars }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.extraEnvVars "context" $) | nindent 14 }}
|
||||
{{- end }}
|
||||
envFrom:
|
||||
{{- if .Values.extraEnvVarsCM }}
|
||||
- configMapRef:
|
||||
name: {{ include "common.tplvalues.render" (dict "value" .Values.extraEnvVarsCM "context" $) }}
|
||||
{{- end }}
|
||||
{{- if .Values.extraEnvVarsSecret }}
|
||||
- secretRef:
|
||||
name: {{ include "common.tplvalues.render" (dict "value" .Values.extraEnvVarsSecret "context" $) }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: client
|
||||
containerPort: {{ .Values.containerPorts.client }}
|
||||
protocol: TCP
|
||||
- name: peer
|
||||
containerPort: {{ .Values.containerPorts.peer }}
|
||||
protocol: TCP
|
||||
{{- if not .Values.diagnosticMode.enabled }}
|
||||
{{- if .Values.customLivenessProbe }}
|
||||
livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customLivenessProbe "context" $) | nindent 14 }}
|
||||
{{- else if .Values.livenessProbe.enabled }}
|
||||
livenessProbe:
|
||||
{{- if .Values.autoCompactionRetention }}
|
||||
- name: ETCD_AUTO_COMPACTION_RETENTION
|
||||
value: {{ .Values.autoCompactionRetention | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.maxProcs }}
|
||||
- name: GOMAXPROCS
|
||||
value: {{ .Values.maxProcs }}
|
||||
{{- end }}
|
||||
{{- if gt $replicaCount 1 }}
|
||||
- name: ETCD_INITIAL_CLUSTER_TOKEN
|
||||
value: "etcd-cluster-k8s"
|
||||
- name: ETCD_INITIAL_CLUSTER_STATE
|
||||
value: {{ default (ternary "new" "existing" .Release.IsInstall) .Values.initialClusterState | quote }}
|
||||
{{- $initialCluster := list }}
|
||||
{{- range $e, $i := until $replicaCount }}
|
||||
{{- $initialCluster = append $initialCluster (printf "%s-%d=%s://%s-%d.%s.%s.svc.%s:%d" $etcdFullname $i $etcdPeerProtocol $etcdFullname $i $etcdHeadlessServiceName $releaseNamespace $clusterDomain $peerPort) }}
|
||||
{{- end }}
|
||||
- name: ETCD_INITIAL_CLUSTER
|
||||
value: {{ join "," $initialCluster | quote }}
|
||||
{{- end }}
|
||||
- name: ETCD_CLUSTER_DOMAIN
|
||||
value: {{ printf "%s.%s.svc.%s" $etcdHeadlessServiceName $releaseNamespace $clusterDomain | quote }}
|
||||
{{- if and .Values.auth.client.secureTransport .Values.auth.client.useAutoTLS }}
|
||||
- name: ETCD_AUTO_TLS
|
||||
value: "true"
|
||||
{{- else if .Values.auth.client.secureTransport }}
|
||||
- name: ETCD_CERT_FILE
|
||||
value: "/opt/bitnami/etcd/certs/client/{{ .Values.auth.client.certFilename }}"
|
||||
- name: ETCD_KEY_FILE
|
||||
value: "/opt/bitnami/etcd/certs/client/{{ .Values.auth.client.certKeyFilename }}"
|
||||
{{- if .Values.auth.client.enableAuthentication }}
|
||||
- name: ETCD_CLIENT_CERT_AUTH
|
||||
value: "true"
|
||||
- name: ETCD_TRUSTED_CA_FILE
|
||||
value: "/opt/bitnami/etcd/certs/client/{{ .Values.auth.client.caFilename | default "ca.crt" }}"
|
||||
{{- else if .Values.auth.client.caFilename }}
|
||||
- name: ETCD_TRUSTED_CA_FILE
|
||||
value: "/opt/bitnami/etcd/certs/client/{{ .Values.auth.client.caFilename | default "ca.crt" }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if and .Values.auth.peer.secureTransport .Values.auth.peer.useAutoTLS }}
|
||||
- name: ETCD_PEER_AUTO_TLS
|
||||
value: "true"
|
||||
{{- else if .Values.auth.peer.secureTransport }}
|
||||
- name: ETCD_PEER_CERT_FILE
|
||||
value: "/opt/bitnami/etcd/certs/peer/{{ .Values.auth.peer.certFilename }}"
|
||||
- name: ETCD_PEER_KEY_FILE
|
||||
value: "/opt/bitnami/etcd/certs/peer/{{ .Values.auth.peer.certKeyFilename }}"
|
||||
{{- if .Values.auth.peer.enableAuthentication }}
|
||||
- name: ETCD_PEER_CLIENT_CERT_AUTH
|
||||
value: "true"
|
||||
- name: ETCD_PEER_TRUSTED_CA_FILE
|
||||
value: "/opt/bitnami/etcd/certs/peer/{{ .Values.auth.peer.caFilename | default "ca.crt" }}"
|
||||
{{- else if .Values.auth.peer.caFilename }}
|
||||
- name: ETCD_PEER_TRUSTED_CA_FILE
|
||||
value: "/opt/bitnami/etcd/certs/peer/{{ .Values.auth.peer.caFilename | default "ca.crt" }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.startFromSnapshot.enabled }}
|
||||
- name: ETCD_INIT_SNAPSHOT_FILENAME
|
||||
value: {{ .Values.startFromSnapshot.snapshotFilename | quote }}
|
||||
- name: ETCD_INIT_SNAPSHOTS_DIR
|
||||
value: {{ ternary "/snapshots" "/init-snapshot" (and .Values.disasterRecovery.enabled (not .Values.disasterRecovery.pvc.existingClaim)) | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.extraEnvVars }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.extraEnvVars "context" $) | nindent 14 }}
|
||||
{{- end }}
|
||||
envFrom:
|
||||
{{- if .Values.extraEnvVarsCM }}
|
||||
- configMapRef:
|
||||
name: {{ include "common.tplvalues.render" (dict "value" .Values.extraEnvVarsCM "context" $) }}
|
||||
{{- end }}
|
||||
{{- if .Values.extraEnvVarsSecret }}
|
||||
- secretRef:
|
||||
name: {{ include "common.tplvalues.render" (dict "value" .Values.extraEnvVarsSecret "context" $) }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: client
|
||||
containerPort: {{ .Values.containerPorts.client }}
|
||||
protocol: TCP
|
||||
- name: peer
|
||||
containerPort: {{ .Values.containerPorts.peer }}
|
||||
protocol: TCP
|
||||
{{- if not .Values.diagnosticMode.enabled }}
|
||||
{{- if .Values.customLivenessProbe }}
|
||||
livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customLivenessProbe "context" $) | nindent 14 }}
|
||||
{{- else if .Values.livenessProbe.enabled }}
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /opt/bitnami/scripts/etcd/healthcheck.sh
|
||||
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
|
||||
successThreshold: {{ .Values.livenessProbe.successThreshold }}
|
||||
failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
|
||||
{{- end }}
|
||||
{{- if .Values.customReadinessProbe }}
|
||||
readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customReadinessProbe "context" $) | nindent 14 }}
|
||||
{{- else if .Values.readinessProbe.enabled }}
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /opt/bitnami/scripts/etcd/healthcheck.sh
|
||||
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
|
||||
successThreshold: {{ .Values.readinessProbe.successThreshold }}
|
||||
failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
|
||||
{{- end }}
|
||||
{{- if .Values.customStartupProbe }}
|
||||
startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customStartupProbe "context" $) | nindent 14 }}
|
||||
{{- else if .Values.startupProbe.enabled }}
|
||||
startupProbe:
|
||||
exec:
|
||||
command:
|
||||
- /opt/bitnami/scripts/etcd/healthcheck.sh
|
||||
initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.startupProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}
|
||||
successThreshold: {{ .Values.startupProbe.successThreshold }}
|
||||
failureThreshold: {{ .Values.startupProbe.failureThreshold }}
|
||||
{{- end }}
|
||||
{{- if .Values.lifecycleHooks }}
|
||||
lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.lifecycleHooks "context" $) | nindent 14 }}
|
||||
{{- else if and (gt $replicaCount 1) .Values.removeMemberOnContainerTermination }}
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
command:
|
||||
- /opt/bitnami/scripts/etcd/healthcheck.sh
|
||||
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
|
||||
successThreshold: {{ .Values.livenessProbe.successThreshold }}
|
||||
failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
|
||||
- /opt/bitnami/scripts/etcd/prestop.sh
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.resources }}
|
||||
resources: {{- include "common.tplvalues.render" (dict "value" .Values.resources "context" $) | nindent 14 }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /bitnami/etcd
|
||||
{{- if and (eq .Values.auth.token.enabled true) (eq .Values.auth.token.type "jwt") }}
|
||||
- name: etcd-jwt-token
|
||||
mountPath: /opt/bitnami/etcd/certs/token/
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if .Values.customReadinessProbe }}
|
||||
readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customReadinessProbe "context" $) | nindent 14 }}
|
||||
{{- else if .Values.readinessProbe.enabled }}
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /opt/bitnami/scripts/etcd/healthcheck.sh
|
||||
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
|
||||
successThreshold: {{ .Values.readinessProbe.successThreshold }}
|
||||
failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
|
||||
{{- if or (and .Values.startFromSnapshot.enabled (not .Values.disasterRecovery.enabled)) (and .Values.disasterRecovery.enabled .Values.startFromSnapshot.enabled .Values.disasterRecovery.pvc.existingClaim) }}
|
||||
- name: init-snapshot-volume
|
||||
mountPath: /init-snapshot
|
||||
{{- end }}
|
||||
{{- if .Values.customStartupProbe }}
|
||||
startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customStartupProbe "context" $) | nindent 14 }}
|
||||
{{- else if .Values.startupProbe.enabled }}
|
||||
startupProbe:
|
||||
exec:
|
||||
command:
|
||||
- /opt/bitnami/scripts/etcd/healthcheck.sh
|
||||
initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.startupProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}
|
||||
successThreshold: {{ .Values.startupProbe.successThreshold }}
|
||||
failureThreshold: {{ .Values.startupProbe.failureThreshold }}
|
||||
{{- if or .Values.disasterRecovery.enabled (and .Values.disasterRecovery.enabled .Values.startFromSnapshot.enabled) }}
|
||||
- name: snapshot-volume
|
||||
mountPath: /snapshots
|
||||
{{- if .Values.disasterRecovery.pvc.subPath }}
|
||||
subPath: {{ .Values.disasterRecovery.pvc.subPath }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.lifecycleHooks }}
|
||||
lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.lifecycleHooks "context" $) | nindent 14 }}
|
||||
{{- else if and (gt $replicaCount 1) .Values.removeMemberOnContainerTermination }}
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
command:
|
||||
- /opt/bitnami/scripts/etcd/prestop.sh
|
||||
{{- if or .Values.configuration .Values.existingConfigmap }}
|
||||
- name: etcd-config
|
||||
mountPath: /opt/bitnami/etcd/conf/
|
||||
{{- end }}
|
||||
{{- if or .Values.auth.client.enableAuthentication (and .Values.auth.client.secureTransport (not .Values.auth.client.useAutoTLS )) }}
|
||||
- name: etcd-client-certs
|
||||
mountPath: /opt/bitnami/etcd/certs/client/
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if .Values.resources }}
|
||||
resources: {{- include "common.tplvalues.render" (dict "value" .Values.resources "context" $) | nindent 14 }}
|
||||
{{- if or .Values.auth.peer.enableAuthentication (and .Values.auth.peer.secureTransport (not .Values.auth.peer.useAutoTLS )) }}
|
||||
- name: etcd-peer-certs
|
||||
mountPath: /opt/bitnami/etcd/certs/peer/
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if .Values.extraVolumeMounts }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.extraVolumeMounts "context" $) | nindent 12 }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /bitnami/etcd
|
||||
{{- if and (eq .Values.auth.token.enabled true) (eq .Values.auth.token.type "jwt") }}
|
||||
- name: etcd-jwt-token
|
||||
mountPath: /opt/bitnami/etcd/certs/token/
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if or (and .Values.startFromSnapshot.enabled (not .Values.disasterRecovery.enabled)) (and .Values.disasterRecovery.enabled .Values.startFromSnapshot.enabled .Values.disasterRecovery.pvc.existingClaim) }}
|
||||
- name: init-snapshot-volume
|
||||
mountPath: /init-snapshot
|
||||
{{- end }}
|
||||
{{- if or .Values.disasterRecovery.enabled (and .Values.disasterRecovery.enabled .Values.startFromSnapshot.enabled) }}
|
||||
- name: snapshot-volume
|
||||
mountPath: /snapshots
|
||||
{{- if .Values.disasterRecovery.pvc.subPath }}
|
||||
subPath: {{ .Values.disasterRecovery.pvc.subPath }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if or .Values.configuration .Values.existingConfigmap }}
|
||||
- name: etcd-config
|
||||
mountPath: /opt/bitnami/etcd/conf/
|
||||
{{- end }}
|
||||
{{- if or .Values.auth.client.enableAuthentication (and .Values.auth.client.secureTransport (not .Values.auth.client.useAutoTLS )) }}
|
||||
- name: etcd-client-certs
|
||||
mountPath: /opt/bitnami/etcd/certs/client/
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if or .Values.auth.peer.enableAuthentication (and .Values.auth.peer.secureTransport (not .Values.auth.peer.useAutoTLS )) }}
|
||||
- name: etcd-peer-certs
|
||||
mountPath: /opt/bitnami/etcd/certs/peer/
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if .Values.extraVolumeMounts }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.extraVolumeMounts "context" $) | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.sidecars }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.sidecars "context" $) | nindent 10 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if and (eq .Values.auth.token.enabled true) (eq .Values.auth.token.type "jwt") }}
|
||||
|
||||
@@ -17,7 +17,7 @@ spec:
|
||||
- {{ . | quote }}
|
||||
{{- end }}
|
||||
persistentVolumeReclaimPolicy: Delete
|
||||
storageClassName: {{ include "common.storage.class" (dict "persistence" .Values.persistence "global" .Values.global) }}
|
||||
{{ include "common.storage.class" (dict "persistence" .Values.persistence "global" .Values.global) }}
|
||||
local: # local类型
|
||||
path: {{ .Values.persistence.storagePath | default "/opt/milvus/data" }}/etcd
|
||||
claimRef:
|
||||
|
||||
@@ -38,6 +38,8 @@ export default function CleansingTaskCreate() {
|
||||
...item.defaultParams,
|
||||
...item.overrides,
|
||||
},
|
||||
inputs: item.inputs,
|
||||
outputs: item.outputs,
|
||||
})),
|
||||
};
|
||||
navigate("/data/cleansing?view=task");
|
||||
|
||||
@@ -44,6 +44,8 @@ export default function CleansingTemplateCreate() {
|
||||
...item.defaultParams,
|
||||
...item.overrides,
|
||||
},
|
||||
inputs: item.inputs,
|
||||
outputs: item.outputs,
|
||||
})),
|
||||
};
|
||||
|
||||
|
||||
@@ -107,7 +107,7 @@ export default function FileTable({result, fetchTaskResult}) {
|
||||
onFilter: (value: string, record: any) =>
|
||||
record.srcName.toLowerCase().includes(value.toLowerCase()),
|
||||
render: (text: string) => (
|
||||
<span className="font-mono text-sm">{text?.replace(/\.[^/.]+$/, "")}</span>
|
||||
<span>{text?.replace(/\.[^/.]+$/, "")}</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
|
||||
@@ -45,6 +45,13 @@ export default function TemplateList() {
|
||||
};
|
||||
|
||||
const templateColumns = [
|
||||
{
|
||||
title: "模板ID",
|
||||
dataIndex: "id",
|
||||
key: "id",
|
||||
fixed: "left",
|
||||
width: 100,
|
||||
},
|
||||
{
|
||||
title: "模板名称",
|
||||
dataIndex: "name",
|
||||
|
||||
@@ -70,7 +70,7 @@ INSERT IGNORE INTO t_operator
|
||||
VALUES ('TextFormatter', 'TXT文本抽取', '抽取TXT中的文本。', '1.0.0', 'text', 'text', null, null, '', false),
|
||||
('UnstructuredFormatter', 'Unstructured文本抽取', '基于Unstructured抽取非结构化文件的文本，目前支持PowerPoint演示文稿、Word文档以及Excel工作簿。', '1.0.0', 'text', 'text', null, null, '', false),
|
||||
('MineruFormatter', 'MinerU PDF文本抽取', '基于MinerU API，抽取PDF中的文本。', '1.0.0', 'text', 'text', null, null, '', false),
|
||||
('FileExporter', '落盘算子', '将文件保存到本地目录。', '1.0.0', 'all', 'all', null, null, '', false),
|
||||
('FileExporter', '落盘算子', '将文件保存到本地目录。', '1.0.0', 'multimodal', 'multimodal', null, null, '', false),
|
||||
('FileWithHighRepeatPhraseRateFilter', '文档词重复率检查', '去除重复词过多的文档。', '1.0.0', 'text', 'text', null, '{"repeatPhraseRatio": {"name": "文档词重复率", "description": "某个词的统计数/文档总词数 > 设定值，该文档被去除。", "type": "slider", "defaultVal": 0.5, "min": 0, "max": 1, "step": 0.1}, "hitStopwords": {"name": "去除停用词", "description": "统计重复词时，选择是否要去除停用词。", "type": "switch", "defaultVal": false, "required": true, "checkedLabel": "去除", "unCheckedLabel": "不去除"}}', '', 'false'),
|
||||
('FileWithHighRepeatWordRateFilter', '文档字重复率检查', '去除重复字过多的文档。', '1.0.0', 'text', 'text', null, '{"repeatWordRatio": {"name": "文档字重复率", "description": "某个字的统计数/文档总字数 > 设定值，该文档被去除。", "type": "slider", "defaultVal": 0.5, "min": 0, "max": 1, "step": 0.1}}', '', 'false'),
|
||||
('FileWithHighSpecialCharRateFilter', '文档特殊字符率检查', '去除特殊字符过多的文档。', '1.0.0', 'text', 'text', null, '{"specialCharRatio": {"name": "文档特殊字符率", "description": "特殊字符的统计数/文档总字数 > 设定值，该文档被去除。", "type": "slider", "defaultVal": 0.3, "min": 0, "max": 1, "step": 0.1}}', '', 'false'),
|
||||
|
||||
Reference in New Issue
Block a user