diff --git a/Makefile b/Makefile index 0ff4192..3829682 100644 --- a/Makefile +++ b/Makefile @@ -209,7 +209,7 @@ endif .PHONY: uninstall uninstall: ifeq ($(origin INSTALLER), undefined) - $(call prompt-uninstaller,milvus-$$INSTALLER-uninstall label-studio-$$INSTALLER-uninstall datamate-$$INSTALLER-uninstall) + $(call prompt-uninstaller,milvus-$$INSTALLER-uninstall datamate-$$INSTALLER-uninstall) else @echo "Delete volumes? (This will remove all data)"; \ echo "1. Yes - Delete volumes"; \ diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTaskService.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTaskService.java index 7e730de..3eba706 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTaskService.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTaskService.java @@ -248,4 +248,8 @@ public class CleaningTaskService { public void stopTask(String taskId) { taskScheduler.stopTask(taskId); } + + public List getInstanceByTemplateId(String templateId) { + return operatorInstanceRepo.findInstanceByInstanceId(templateId); + } } diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTemplateService.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTemplateService.java index cc20169..1303137 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTemplateService.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTemplateService.java @@ -3,6 +3,7 @@ package com.datamate.cleaning.application; import com.datamate.cleaning.domain.repository.CleaningTemplateRepository; import com.datamate.cleaning.domain.repository.OperatorInstanceRepository; +import com.datamate.cleaning.infrastructure.validator.CleanTaskValidator; import com.datamate.cleaning.interfaces.dto.*; import com.datamate.cleaning.domain.model.entity.TemplateWithInstance; import com.datamate.operator.domain.repository.OperatorViewRepository; @@ -28,6 +29,8 @@ public class CleaningTemplateService { private final OperatorViewRepository operatorViewRepo; + private final CleanTaskValidator cleanTaskValidator; + public List getTemplates(String keywords) { List allOperators = operatorViewRepo.findOperatorsByCriteria(null, null, null, null, null); @@ -59,6 +62,7 @@ public class CleaningTemplateService { @Transactional public CleaningTemplateDto createTemplate(CreateCleaningTemplateRequest request) { + cleanTaskValidator.checkInputAndOutput(request.getInstance()); CleaningTemplateDto template = new CleaningTemplateDto(); String templateId = UUID.randomUUID().toString(); template.setId(templateId); diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/repository/OperatorInstanceRepository.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/repository/OperatorInstanceRepository.java index c956c17..9d1900e 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/repository/OperatorInstanceRepository.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/repository/OperatorInstanceRepository.java @@ -13,4 +13,6 @@ public interface OperatorInstanceRepository extends IRepository findOperatorByInstanceId(String instanceId); + + List findInstanceByInstanceId(String instanceId); } diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/converter/OperatorInstanceConverter.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/converter/OperatorInstanceConverter.java index 556a4d5..3cb47f3 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/converter/OperatorInstanceConverter.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/converter/OperatorInstanceConverter.java @@ -8,6 +8,7 @@ import com.datamate.common.infrastructure.exception.SystemErrorCode; import com.datamate.operator.domain.model.OperatorView; import com.datamate.operator.interfaces.dto.OperatorDto; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.mapstruct.Mapper; import org.mapstruct.Mapping; @@ -23,20 +24,39 @@ import java.util.Map; public interface OperatorInstanceConverter { OperatorInstanceConverter INSTANCE = Mappers.getMapper(OperatorInstanceConverter.class); + ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + @Mapping(target = "settingsOverride", source = "overrides", qualifiedByName = "mapToString") @Mapping(target = "operatorId", source = "id") OperatorInstance fromDtoToEntity(OperatorInstanceDto instance); + @Mapping(target = "overrides", source = "settingsOverride", qualifiedByName = "stringToMap") + @Mapping(target = "id", source = "operatorId") + OperatorInstanceDto fromEntityToDto(OperatorInstance instance); + + List fromEntityToDtoList(List instance); + @Named("mapToString") static String mapToString(Map objects) { - ObjectMapper objectMapper = new ObjectMapper(); try { - return objectMapper.writeValueAsString(objects); + return OBJECT_MAPPER.writeValueAsString(objects); } catch (JsonProcessingException e) { throw BusinessException.of(SystemErrorCode.UNKNOWN_ERROR); } } + @Named("stringToMap") + static Map stringToMap(String json) { + if (json == null) { + return Collections.emptyMap(); + } + try { + return OBJECT_MAPPER.readValue(json, new TypeReference<>() {}); + } catch (JsonProcessingException e) { + throw BusinessException.of(SystemErrorCode.UNKNOWN_ERROR); + } + } + @Mapping(target = "categories", source = "categories", qualifiedByName = "stringToList") OperatorDto fromEntityToDto(OperatorView operator); diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/Impl/OperatorInstanceRepositoryImpl.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/Impl/OperatorInstanceRepositoryImpl.java index e5fe038..813f1e6 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/Impl/OperatorInstanceRepositoryImpl.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/Impl/OperatorInstanceRepositoryImpl.java @@ -42,4 +42,12 @@ public class OperatorInstanceRepositoryImpl extends CrudRepository findOperatorByInstanceId(String instanceId) { return OperatorInstanceConverter.INSTANCE.fromEntityToDto(mapper.findOperatorByInstanceId(instanceId)); } + + @Override + public List findInstanceByInstanceId(String instanceId) { + LambdaQueryWrapper lambdaWrapper = new LambdaQueryWrapper<>(); + lambdaWrapper.eq(OperatorInstance::getInstanceId, instanceId) + .orderByAsc(OperatorInstance::getOpIndex); + return OperatorInstanceConverter.INSTANCE.fromEntityToDtoList(mapper.selectList(lambdaWrapper)); + } } diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/CreateCleaningTaskRequest.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/CreateCleaningTaskRequest.java index 76736cf..b7da4a0 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/CreateCleaningTaskRequest.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/CreateCleaningTaskRequest.java @@ -27,6 +27,8 @@ public class CreateCleaningTaskRequest { private String destDatasetType; + private String templateId; + private List instance = new ArrayList<>(); } diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/rest/CleaningTaskController.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/rest/CleaningTaskController.java index bdca1dc..25ec2e8 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/rest/CleaningTaskController.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/rest/CleaningTaskController.java @@ -1,12 +1,10 @@ package com.datamate.cleaning.interfaces.rest; import com.datamate.cleaning.application.CleaningTaskService; -import com.datamate.cleaning.interfaces.dto.CleaningResultDto; -import com.datamate.cleaning.interfaces.dto.CleaningTaskDto; -import com.datamate.cleaning.interfaces.dto.CleaningTaskLog; -import com.datamate.cleaning.interfaces.dto.CreateCleaningTaskRequest; +import com.datamate.cleaning.interfaces.dto.*; import com.datamate.common.interfaces.PagedResponse; import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; import org.springframework.web.bind.annotation.*; import java.util.List; @@ -31,6 +29,9 @@ public class CleaningTaskController { @PostMapping public CleaningTaskDto cleaningTasksPost(@RequestBody CreateCleaningTaskRequest request) { + if (request.getInstance().isEmpty() && StringUtils.isNotBlank(request.getTemplateId())) { + request.setInstance(cleaningTaskService.getInstanceByTemplateId(request.getTemplateId())); + } return cleaningTaskService.createTask(request); } @@ -57,6 +58,13 @@ public class CleaningTaskController { return taskId; } + @DeleteMapping + public void cleaningTasksDelete(@RequestParam List taskIds) { + for (String taskId : taskIds) { + cleaningTaskService.deleteTask(taskId); + } + } + @GetMapping("/{taskId}/result") public List cleaningTasksTaskIdGetResult(@PathVariable("taskId") String taskId) { return cleaningTaskService.getTaskResults(taskId); diff --git a/deployment/helm/milvus/charts/etcd/templates/deployment.yaml b/deployment/helm/milvus/charts/etcd/templates/deployment.yaml index fd8a546..6166cba 100644 --- a/deployment/helm/milvus/charts/etcd/templates/deployment.yaml +++ b/deployment/helm/milvus/charts/etcd/templates/deployment.yaml @@ -102,261 +102,261 @@ spec: {{- end }} containers: {{- $replicaCount := int .Values.replicaCount }} - {{- $peerPort := int .Values.containerPorts.peer }} - {{- $etcdFullname := include "common.names.fullname" . }} - {{- $releaseNamespace := .Release.Namespace }} - {{- $etcdHeadlessServiceName := (printf "%s-%s" $etcdFullname "headless" | trunc 63 | trimSuffix "-") }} - {{- $clusterDomain := .Values.clusterDomain }} - {{- $etcdPeerProtocol := include "etcd.peerProtocol" . }} - {{- $etcdClientProtocol := include "etcd.clientProtocol" . }} - - name: etcd - image: {{ include "etcd.image" . }} - imagePullPolicy: {{ .Values.image.pullPolicy | quote }} - {{- if .Values.containerSecurityContext.enabled }} - securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- $peerPort := int .Values.containerPorts.peer }} + {{- $etcdFullname := include "common.names.fullname" . }} + {{- $releaseNamespace := .Release.Namespace }} + {{- $etcdHeadlessServiceName := (printf "%s-%s" $etcdFullname "headless" | trunc 63 | trimSuffix "-") }} + {{- $clusterDomain := .Values.clusterDomain }} + {{- $etcdPeerProtocol := include "etcd.peerProtocol" . }} + {{- $etcdClientProtocol := include "etcd.clientProtocol" . }} + - name: etcd + image: {{ include "etcd.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 14 }} + {{- end }} + {{- if .Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }} + {{- else if .Values.command }} + command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 14 }} + {{- end }} + {{- if .Values.diagnosticMode.enabled }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }} + {{- else if .Values.args }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 14 }} + {{- end }} + env: + - name: BITNAMI_DEBUG + value: {{ ternary "true" "false" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_STS_NAME + value: {{ include "common.names.fullname" . | quote }} + - name: ETCDCTL_API + value: "3" + - name: ETCD_ON_K8S + value: "yes" + - name: ETCD_START_FROM_SNAPSHOT + value: {{ ternary "yes" "no" .Values.startFromSnapshot.enabled | quote }} + - name: ETCD_DISASTER_RECOVERY + value: {{ ternary "yes" "no" .Values.disasterRecovery.enabled | quote }} + - name: ETCD_NAME + value: "$(MY_POD_NAME)" + - name: ETCD_DATA_DIR + value: "/bitnami/etcd/data" + - name: ETCD_LOG_LEVEL + value: {{ ternary "debug" .Values.logLevel .Values.image.debug | quote }} + - name: ALLOW_NONE_AUTHENTICATION + value: {{ ternary "yes" "no" (and (not (or .Values.auth.rbac.create .Values.auth.rbac.enabled)) .Values.auth.rbac.allowNoneAuthentication) | quote }} + {{- if or .Values.auth.rbac.create .Values.auth.rbac.enabled }} + - name: ETCD_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "etcd.secretName" . }} + key: {{ include "etcd.secretPasswordKey" . }} {{- end }} - {{- if .Values.diagnosticMode.enabled }} - command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }} - {{- else if .Values.command }} - command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 14 }} + {{- if .Values.auth.token.enabled }} + - name: ETCD_AUTH_TOKEN + {{- if eq .Values.auth.token.type "jwt" }} + value: {{ printf "jwt,priv-key=/opt/bitnami/etcd/certs/token/%s,sign-method=%s,ttl=%s" .Values.auth.token.privateKey.filename .Values.auth.token.signMethod .Values.auth.token.ttl | quote }} + {{- else if eq .Values.auth.token.type "simple" }} + value: "simple" + {{- end }} {{- end }} - {{- if .Values.diagnosticMode.enabled }} - args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }} - {{- else if .Values.args }} - args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 14 }} + # 警告:以下 Env Vars 依赖 StatefulSet 命名 (headless service 和 ordered names) + # 在 Deployment 中如果不修改副本数为 1,这些配置会导致集群发现失败。 + - name: ETCD_ADVERTISE_CLIENT_URLS + value: "{{ $etcdClientProtocol }}://$(MY_POD_NAME).{{ $etcdHeadlessServiceName }}.{{ .Release.Namespace }}.svc.{{ $clusterDomain }}:{{ .Values.containerPorts.client }},{{ $etcdClientProtocol }}://{{ $etcdFullname }}.{{ .Release.Namespace }}.svc.{{ $clusterDomain }}:{{ coalesce .Values.service.ports.client .Values.service.port }}" + - name: ETCD_LISTEN_CLIENT_URLS + value: "{{ $etcdClientProtocol }}://0.0.0.0:{{ .Values.containerPorts.client }}" + - name: ETCD_INITIAL_ADVERTISE_PEER_URLS + value: "{{ $etcdPeerProtocol }}://$(MY_POD_NAME).{{ $etcdHeadlessServiceName }}.{{ .Release.Namespace }}.svc.{{ $clusterDomain }}:{{ .Values.containerPorts.peer }}" + - name: ETCD_LISTEN_PEER_URLS + value: "{{ $etcdPeerProtocol }}://0.0.0.0:{{ .Values.containerPorts.peer }}" + {{- if .Values.autoCompactionMode }} + - name: ETCD_AUTO_COMPACTION_MODE + value: {{ .Values.autoCompactionMode | quote }} {{- end }} - env: - - name: BITNAMI_DEBUG - value: {{ ternary "true" "false" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: MY_STS_NAME - value: {{ include "common.names.fullname" . | quote }} - - name: ETCDCTL_API - value: "3" - - name: ETCD_ON_K8S - value: "yes" - - name: ETCD_START_FROM_SNAPSHOT - value: {{ ternary "yes" "no" .Values.startFromSnapshot.enabled | quote }} - - name: ETCD_DISASTER_RECOVERY - value: {{ ternary "yes" "no" .Values.disasterRecovery.enabled | quote }} - - name: ETCD_NAME - value: "$(MY_POD_NAME)" - - name: ETCD_DATA_DIR - value: "/bitnami/etcd/data" - - name: ETCD_LOG_LEVEL - value: {{ ternary "debug" .Values.logLevel .Values.image.debug | quote }} - - name: ALLOW_NONE_AUTHENTICATION - value: {{ ternary "yes" "no" (and (not (or .Values.auth.rbac.create .Values.auth.rbac.enabled)) .Values.auth.rbac.allowNoneAuthentication) | quote }} - {{- if or .Values.auth.rbac.create .Values.auth.rbac.enabled }} - - name: ETCD_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "etcd.secretName" . }} - key: {{ include "etcd.secretPasswordKey" . }} - {{- end }} - {{- if .Values.auth.token.enabled }} - - name: ETCD_AUTH_TOKEN - {{- if eq .Values.auth.token.type "jwt" }} - value: {{ printf "jwt,priv-key=/opt/bitnami/etcd/certs/token/%s,sign-method=%s,ttl=%s" .Values.auth.token.privateKey.filename .Values.auth.token.signMethod .Values.auth.token.ttl | quote }} - {{- else if eq .Values.auth.token.type "simple" }} - value: "simple" - {{- end }} - {{- end }} - # 警告:以下 Env Vars 依赖 StatefulSet 命名 (headless service 和 ordered names) - # 在 Deployment 中如果不修改副本数为 1,这些配置会导致集群发现失败。 - - name: ETCD_ADVERTISE_CLIENT_URLS - value: "{{ $etcdClientProtocol }}://$(MY_POD_NAME).{{ $etcdHeadlessServiceName }}.{{ .Release.Namespace }}.svc.{{ $clusterDomain }}:{{ .Values.containerPorts.client }},{{ $etcdClientProtocol }}://{{ $etcdFullname }}.{{ .Release.Namespace }}.svc.{{ $clusterDomain }}:{{ coalesce .Values.service.ports.client .Values.service.port }}" - - name: ETCD_LISTEN_CLIENT_URLS - value: "{{ $etcdClientProtocol }}://0.0.0.0:{{ .Values.containerPorts.client }}" - - name: ETCD_INITIAL_ADVERTISE_PEER_URLS - value: "{{ $etcdPeerProtocol }}://$(MY_POD_NAME).{{ $etcdHeadlessServiceName }}.{{ .Release.Namespace }}.svc.{{ $clusterDomain }}:{{ .Values.containerPorts.peer }}" - - name: ETCD_LISTEN_PEER_URLS - value: "{{ $etcdPeerProtocol }}://0.0.0.0:{{ .Values.containerPorts.peer }}" - {{- if .Values.autoCompactionMode }} - - name: ETCD_AUTO_COMPACTION_MODE - value: {{ .Values.autoCompactionMode | quote }} - {{- end }} - {{- if .Values.autoCompactionRetention }} - - name: ETCD_AUTO_COMPACTION_RETENTION - value: {{ .Values.autoCompactionRetention | quote }} - {{- end }} - {{- if .Values.maxProcs }} - - name: GOMAXPROCS - value: {{ .Values.maxProcs }} - {{- end }} - {{- if gt $replicaCount 1 }} - - name: ETCD_INITIAL_CLUSTER_TOKEN - value: "etcd-cluster-k8s" - - name: ETCD_INITIAL_CLUSTER_STATE - value: {{ default (ternary "new" "existing" .Release.IsInstall) .Values.initialClusterState | quote }} - {{- $initialCluster := list }} - {{- range $e, $i := until $replicaCount }} - {{- $initialCluster = append $initialCluster (printf "%s-%d=%s://%s-%d.%s.%s.svc.%s:%d" $etcdFullname $i $etcdPeerProtocol $etcdFullname $i $etcdHeadlessServiceName $releaseNamespace $clusterDomain $peerPort) }} - {{- end }} - - name: ETCD_INITIAL_CLUSTER - value: {{ join "," $initialCluster | quote }} - {{- end }} - - name: ETCD_CLUSTER_DOMAIN - value: {{ printf "%s.%s.svc.%s" $etcdHeadlessServiceName $releaseNamespace $clusterDomain | quote }} - {{- if and .Values.auth.client.secureTransport .Values.auth.client.useAutoTLS }} - - name: ETCD_AUTO_TLS - value: "true" - {{- else if .Values.auth.client.secureTransport }} - - name: ETCD_CERT_FILE - value: "/opt/bitnami/etcd/certs/client/{{ .Values.auth.client.certFilename }}" - - name: ETCD_KEY_FILE - value: "/opt/bitnami/etcd/certs/client/{{ .Values.auth.client.certKeyFilename }}" - {{- if .Values.auth.client.enableAuthentication }} - - name: ETCD_CLIENT_CERT_AUTH - value: "true" - - name: ETCD_TRUSTED_CA_FILE - value: "/opt/bitnami/etcd/certs/client/{{ .Values.auth.client.caFilename | default "ca.crt" }}" - {{- else if .Values.auth.client.caFilename }} - - name: ETCD_TRUSTED_CA_FILE - value: "/opt/bitnami/etcd/certs/client/{{ .Values.auth.client.caFilename | default "ca.crt" }}" - {{- end }} - {{- end }} - {{- if and .Values.auth.peer.secureTransport .Values.auth.peer.useAutoTLS }} - - name: ETCD_PEER_AUTO_TLS - value: "true" - {{- else if .Values.auth.peer.secureTransport }} - - name: ETCD_PEER_CERT_FILE - value: "/opt/bitnami/etcd/certs/peer/{{ .Values.auth.peer.certFilename }}" - - name: ETCD_PEER_KEY_FILE - value: "/opt/bitnami/etcd/certs/peer/{{ .Values.auth.peer.certKeyFilename }}" - {{- if .Values.auth.peer.enableAuthentication }} - - name: ETCD_PEER_CLIENT_CERT_AUTH - value: "true" - - name: ETCD_PEER_TRUSTED_CA_FILE - value: "/opt/bitnami/etcd/certs/peer/{{ .Values.auth.peer.caFilename | default "ca.crt" }}" - {{- else if .Values.auth.peer.caFilename }} - - name: ETCD_PEER_TRUSTED_CA_FILE - value: "/opt/bitnami/etcd/certs/peer/{{ .Values.auth.peer.caFilename | default "ca.crt" }}" - {{- end }} - {{- end }} - {{- if .Values.startFromSnapshot.enabled }} - - name: ETCD_INIT_SNAPSHOT_FILENAME - value: {{ .Values.startFromSnapshot.snapshotFilename | quote }} - - name: ETCD_INIT_SNAPSHOTS_DIR - value: {{ ternary "/snapshots" "/init-snapshot" (and .Values.disasterRecovery.enabled (not .Values.disasterRecovery.pvc.existingClaim)) | quote }} - {{- end }} - {{- if .Values.extraEnvVars }} - {{- include "common.tplvalues.render" (dict "value" .Values.extraEnvVars "context" $) | nindent 14 }} - {{- end }} - envFrom: - {{- if .Values.extraEnvVarsCM }} - - configMapRef: - name: {{ include "common.tplvalues.render" (dict "value" .Values.extraEnvVarsCM "context" $) }} - {{- end }} - {{- if .Values.extraEnvVarsSecret }} - - secretRef: - name: {{ include "common.tplvalues.render" (dict "value" .Values.extraEnvVarsSecret "context" $) }} - {{- end }} - ports: - - name: client - containerPort: {{ .Values.containerPorts.client }} - protocol: TCP - - name: peer - containerPort: {{ .Values.containerPorts.peer }} - protocol: TCP - {{- if not .Values.diagnosticMode.enabled }} - {{- if .Values.customLivenessProbe }} - livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customLivenessProbe "context" $) | nindent 14 }} - {{- else if .Values.livenessProbe.enabled }} - livenessProbe: + {{- if .Values.autoCompactionRetention }} + - name: ETCD_AUTO_COMPACTION_RETENTION + value: {{ .Values.autoCompactionRetention | quote }} + {{- end }} + {{- if .Values.maxProcs }} + - name: GOMAXPROCS + value: {{ .Values.maxProcs }} + {{- end }} + {{- if gt $replicaCount 1 }} + - name: ETCD_INITIAL_CLUSTER_TOKEN + value: "etcd-cluster-k8s" + - name: ETCD_INITIAL_CLUSTER_STATE + value: {{ default (ternary "new" "existing" .Release.IsInstall) .Values.initialClusterState | quote }} + {{- $initialCluster := list }} + {{- range $e, $i := until $replicaCount }} + {{- $initialCluster = append $initialCluster (printf "%s-%d=%s://%s-%d.%s.%s.svc.%s:%d" $etcdFullname $i $etcdPeerProtocol $etcdFullname $i $etcdHeadlessServiceName $releaseNamespace $clusterDomain $peerPort) }} + {{- end }} + - name: ETCD_INITIAL_CLUSTER + value: {{ join "," $initialCluster | quote }} + {{- end }} + - name: ETCD_CLUSTER_DOMAIN + value: {{ printf "%s.%s.svc.%s" $etcdHeadlessServiceName $releaseNamespace $clusterDomain | quote }} + {{- if and .Values.auth.client.secureTransport .Values.auth.client.useAutoTLS }} + - name: ETCD_AUTO_TLS + value: "true" + {{- else if .Values.auth.client.secureTransport }} + - name: ETCD_CERT_FILE + value: "/opt/bitnami/etcd/certs/client/{{ .Values.auth.client.certFilename }}" + - name: ETCD_KEY_FILE + value: "/opt/bitnami/etcd/certs/client/{{ .Values.auth.client.certKeyFilename }}" + {{- if .Values.auth.client.enableAuthentication }} + - name: ETCD_CLIENT_CERT_AUTH + value: "true" + - name: ETCD_TRUSTED_CA_FILE + value: "/opt/bitnami/etcd/certs/client/{{ .Values.auth.client.caFilename | default "ca.crt" }}" + {{- else if .Values.auth.client.caFilename }} + - name: ETCD_TRUSTED_CA_FILE + value: "/opt/bitnami/etcd/certs/client/{{ .Values.auth.client.caFilename | default "ca.crt" }}" + {{- end }} + {{- end }} + {{- if and .Values.auth.peer.secureTransport .Values.auth.peer.useAutoTLS }} + - name: ETCD_PEER_AUTO_TLS + value: "true" + {{- else if .Values.auth.peer.secureTransport }} + - name: ETCD_PEER_CERT_FILE + value: "/opt/bitnami/etcd/certs/peer/{{ .Values.auth.peer.certFilename }}" + - name: ETCD_PEER_KEY_FILE + value: "/opt/bitnami/etcd/certs/peer/{{ .Values.auth.peer.certKeyFilename }}" + {{- if .Values.auth.peer.enableAuthentication }} + - name: ETCD_PEER_CLIENT_CERT_AUTH + value: "true" + - name: ETCD_PEER_TRUSTED_CA_FILE + value: "/opt/bitnami/etcd/certs/peer/{{ .Values.auth.peer.caFilename | default "ca.crt" }}" + {{- else if .Values.auth.peer.caFilename }} + - name: ETCD_PEER_TRUSTED_CA_FILE + value: "/opt/bitnami/etcd/certs/peer/{{ .Values.auth.peer.caFilename | default "ca.crt" }}" + {{- end }} + {{- end }} + {{- if .Values.startFromSnapshot.enabled }} + - name: ETCD_INIT_SNAPSHOT_FILENAME + value: {{ .Values.startFromSnapshot.snapshotFilename | quote }} + - name: ETCD_INIT_SNAPSHOTS_DIR + value: {{ ternary "/snapshots" "/init-snapshot" (and .Values.disasterRecovery.enabled (not .Values.disasterRecovery.pvc.existingClaim)) | quote }} + {{- end }} + {{- if .Values.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.extraEnvVars "context" $) | nindent 14 }} + {{- end }} + envFrom: + {{- if .Values.extraEnvVarsCM }} + - configMapRef: + name: {{ include "common.tplvalues.render" (dict "value" .Values.extraEnvVarsCM "context" $) }} + {{- end }} + {{- if .Values.extraEnvVarsSecret }} + - secretRef: + name: {{ include "common.tplvalues.render" (dict "value" .Values.extraEnvVarsSecret "context" $) }} + {{- end }} + ports: + - name: client + containerPort: {{ .Values.containerPorts.client }} + protocol: TCP + - name: peer + containerPort: {{ .Values.containerPorts.peer }} + protocol: TCP + {{- if not .Values.diagnosticMode.enabled }} + {{- if .Values.customLivenessProbe }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customLivenessProbe "context" $) | nindent 14 }} + {{- else if .Values.livenessProbe.enabled }} + livenessProbe: + exec: + command: + - /opt/bitnami/scripts/etcd/healthcheck.sh + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.customReadinessProbe }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customReadinessProbe "context" $) | nindent 14 }} + {{- else if .Values.readinessProbe.enabled }} + readinessProbe: + exec: + command: + - /opt/bitnami/scripts/etcd/healthcheck.sh + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.customStartupProbe }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customStartupProbe "context" $) | nindent 14 }} + {{- else if .Values.startupProbe.enabled }} + startupProbe: + exec: + command: + - /opt/bitnami/scripts/etcd/healthcheck.sh + initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.startupProbe.periodSeconds }} + timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }} + successThreshold: {{ .Values.startupProbe.successThreshold }} + failureThreshold: {{ .Values.startupProbe.failureThreshold }} + {{- end }} + {{- if .Values.lifecycleHooks }} + lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.lifecycleHooks "context" $) | nindent 14 }} + {{- else if and (gt $replicaCount 1) .Values.removeMemberOnContainerTermination }} + lifecycle: + preStop: exec: command: - - /opt/bitnami/scripts/etcd/healthcheck.sh - initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.livenessProbe.periodSeconds }} - timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} - successThreshold: {{ .Values.livenessProbe.successThreshold }} - failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + - /opt/bitnami/scripts/etcd/prestop.sh + {{- end }} + {{- end }} + {{- if .Values.resources }} + resources: {{- include "common.tplvalues.render" (dict "value" .Values.resources "context" $) | nindent 14 }} + {{- end }} + volumeMounts: + - name: data + mountPath: /bitnami/etcd + {{- if and (eq .Values.auth.token.enabled true) (eq .Values.auth.token.type "jwt") }} + - name: etcd-jwt-token + mountPath: /opt/bitnami/etcd/certs/token/ + readOnly: true {{- end }} - {{- if .Values.customReadinessProbe }} - readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customReadinessProbe "context" $) | nindent 14 }} - {{- else if .Values.readinessProbe.enabled }} - readinessProbe: - exec: - command: - - /opt/bitnami/scripts/etcd/healthcheck.sh - initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.readinessProbe.periodSeconds }} - timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} - successThreshold: {{ .Values.readinessProbe.successThreshold }} - failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- if or (and .Values.startFromSnapshot.enabled (not .Values.disasterRecovery.enabled)) (and .Values.disasterRecovery.enabled .Values.startFromSnapshot.enabled .Values.disasterRecovery.pvc.existingClaim) }} + - name: init-snapshot-volume + mountPath: /init-snapshot {{- end }} - {{- if .Values.customStartupProbe }} - startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customStartupProbe "context" $) | nindent 14 }} - {{- else if .Values.startupProbe.enabled }} - startupProbe: - exec: - command: - - /opt/bitnami/scripts/etcd/healthcheck.sh - initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.startupProbe.periodSeconds }} - timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }} - successThreshold: {{ .Values.startupProbe.successThreshold }} - failureThreshold: {{ .Values.startupProbe.failureThreshold }} + {{- if or .Values.disasterRecovery.enabled (and .Values.disasterRecovery.enabled .Values.startFromSnapshot.enabled) }} + - name: snapshot-volume + mountPath: /snapshots + {{- if .Values.disasterRecovery.pvc.subPath }} + subPath: {{ .Values.disasterRecovery.pvc.subPath }} + {{- end }} {{- end }} - {{- if .Values.lifecycleHooks }} - lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.lifecycleHooks "context" $) | nindent 14 }} - {{- else if and (gt $replicaCount 1) .Values.removeMemberOnContainerTermination }} - lifecycle: - preStop: - exec: - command: - - /opt/bitnami/scripts/etcd/prestop.sh + {{- if or .Values.configuration .Values.existingConfigmap }} + - name: etcd-config + mountPath: /opt/bitnami/etcd/conf/ {{- end }} + {{- if or .Values.auth.client.enableAuthentication (and .Values.auth.client.secureTransport (not .Values.auth.client.useAutoTLS )) }} + - name: etcd-client-certs + mountPath: /opt/bitnami/etcd/certs/client/ + readOnly: true {{- end }} - {{- if .Values.resources }} - resources: {{- include "common.tplvalues.render" (dict "value" .Values.resources "context" $) | nindent 14 }} + {{- if or .Values.auth.peer.enableAuthentication (and .Values.auth.peer.secureTransport (not .Values.auth.peer.useAutoTLS )) }} + - name: etcd-peer-certs + mountPath: /opt/bitnami/etcd/certs/peer/ + readOnly: true + {{- end }} + {{- if .Values.extraVolumeMounts }} + {{- include "common.tplvalues.render" (dict "value" .Values.extraVolumeMounts "context" $) | nindent 12 }} {{- end }} - volumeMounts: - - name: data - mountPath: /bitnami/etcd - {{- if and (eq .Values.auth.token.enabled true) (eq .Values.auth.token.type "jwt") }} - - name: etcd-jwt-token - mountPath: /opt/bitnami/etcd/certs/token/ - readOnly: true - {{- end }} - {{- if or (and .Values.startFromSnapshot.enabled (not .Values.disasterRecovery.enabled)) (and .Values.disasterRecovery.enabled .Values.startFromSnapshot.enabled .Values.disasterRecovery.pvc.existingClaim) }} - - name: init-snapshot-volume - mountPath: /init-snapshot - {{- end }} - {{- if or .Values.disasterRecovery.enabled (and .Values.disasterRecovery.enabled .Values.startFromSnapshot.enabled) }} - - name: snapshot-volume - mountPath: /snapshots - {{- if .Values.disasterRecovery.pvc.subPath }} - subPath: {{ .Values.disasterRecovery.pvc.subPath }} - {{- end }} - {{- end }} - {{- if or .Values.configuration .Values.existingConfigmap }} - - name: etcd-config - mountPath: /opt/bitnami/etcd/conf/ - {{- end }} - {{- if or .Values.auth.client.enableAuthentication (and .Values.auth.client.secureTransport (not .Values.auth.client.useAutoTLS )) }} - - name: etcd-client-certs - mountPath: /opt/bitnami/etcd/certs/client/ - readOnly: true - {{- end }} - {{- if or .Values.auth.peer.enableAuthentication (and .Values.auth.peer.secureTransport (not .Values.auth.peer.useAutoTLS )) }} - - name: etcd-peer-certs - mountPath: /opt/bitnami/etcd/certs/peer/ - readOnly: true - {{- end }} - {{- if .Values.extraVolumeMounts }} - {{- include "common.tplvalues.render" (dict "value" .Values.extraVolumeMounts "context" $) | nindent 12 }} - {{- end }} {{- if .Values.sidecars }} - {{- include "common.tplvalues.render" (dict "value" .Values.sidecars "context" $) | nindent 8 }} + {{- include "common.tplvalues.render" (dict "value" .Values.sidecars "context" $) | nindent 10 }} {{- end }} volumes: {{- if and (eq .Values.auth.token.enabled true) (eq .Values.auth.token.type "jwt") }} diff --git a/deployment/helm/milvus/charts/etcd/templates/pvc.yaml b/deployment/helm/milvus/charts/etcd/templates/pvc.yaml index cf10b27..e7a73e9 100644 --- a/deployment/helm/milvus/charts/etcd/templates/pvc.yaml +++ b/deployment/helm/milvus/charts/etcd/templates/pvc.yaml @@ -17,7 +17,7 @@ spec: - {{ . | quote }} {{- end }} persistentVolumeReclaimPolicy: Delete - storageClassName: {{ include "common.storage.class" (dict "persistence" .Values.persistence "global" .Values.global) }} + {{ include "common.storage.class" (dict "persistence" .Values.persistence "global" .Values.global) }} local: # local类型 path: {{ .Values.persistence.storagePath | default "/opt/milvus/data" }}/etcd claimRef: diff --git a/frontend/src/pages/DataCleansing/Create/CreateTask.tsx b/frontend/src/pages/DataCleansing/Create/CreateTask.tsx index a3f6d03..b0fbe57 100644 --- a/frontend/src/pages/DataCleansing/Create/CreateTask.tsx +++ b/frontend/src/pages/DataCleansing/Create/CreateTask.tsx @@ -38,6 +38,8 @@ export default function CleansingTaskCreate() { ...item.defaultParams, ...item.overrides, }, + inputs: item.inputs, + outputs: item.outputs, })), }; navigate("/data/cleansing?view=task"); diff --git a/frontend/src/pages/DataCleansing/Create/CreateTemplate.tsx b/frontend/src/pages/DataCleansing/Create/CreateTemplate.tsx index bb67588..ca8694b 100644 --- a/frontend/src/pages/DataCleansing/Create/CreateTemplate.tsx +++ b/frontend/src/pages/DataCleansing/Create/CreateTemplate.tsx @@ -44,6 +44,8 @@ export default function CleansingTemplateCreate() { ...item.defaultParams, ...item.overrides, }, + inputs: item.inputs, + outputs: item.outputs, })), }; diff --git a/frontend/src/pages/DataCleansing/Detail/components/FileTable.tsx b/frontend/src/pages/DataCleansing/Detail/components/FileTable.tsx index d078091..a13ff6a 100644 --- a/frontend/src/pages/DataCleansing/Detail/components/FileTable.tsx +++ b/frontend/src/pages/DataCleansing/Detail/components/FileTable.tsx @@ -107,7 +107,7 @@ export default function FileTable({result, fetchTaskResult}) { onFilter: (value: string, record: any) => record.srcName.toLowerCase().includes(value.toLowerCase()), render: (text: string) => ( - {text?.replace(/\.[^/.]+$/, "")} + {text?.replace(/\.[^/.]+$/, "")} ), }, { diff --git a/frontend/src/pages/DataCleansing/Home/components/TemplateList.tsx b/frontend/src/pages/DataCleansing/Home/components/TemplateList.tsx index e396e92..ed0fdb3 100644 --- a/frontend/src/pages/DataCleansing/Home/components/TemplateList.tsx +++ b/frontend/src/pages/DataCleansing/Home/components/TemplateList.tsx @@ -45,6 +45,13 @@ export default function TemplateList() { }; const templateColumns = [ + { + title: "模板ID", + dataIndex: "id", + key: "id", + fixed: "left", + width: 100, + }, { title: "模板名称", dataIndex: "name", diff --git a/scripts/db/data-operator-init.sql b/scripts/db/data-operator-init.sql index efed70c..172ffbc 100644 --- a/scripts/db/data-operator-init.sql +++ b/scripts/db/data-operator-init.sql @@ -70,7 +70,7 @@ INSERT IGNORE INTO t_operator VALUES ('TextFormatter', 'TXT文本抽取', '抽取TXT中的文本。', '1.0.0', 'text', 'text', null, null, '', false), ('UnstructuredFormatter', 'Unstructured文本抽取', '基于Unstructured抽取非结构化文件的文本,目前支持PowerPoint演示文稿、Word文档以及Excel工作簿。', '1.0.0', 'text', 'text', null, null, '', false), ('MineruFormatter', 'MinerU PDF文本抽取', '基于MinerU API,抽取PDF中的文本。', '1.0.0', 'text', 'text', null, null, '', false), - ('FileExporter', '落盘算子', '将文件保存到本地目录。', '1.0.0', 'all', 'all', null, null, '', false), + ('FileExporter', '落盘算子', '将文件保存到本地目录。', '1.0.0', 'multimodal', 'multimodal', null, null, '', false), ('FileWithHighRepeatPhraseRateFilter', '文档词重复率检查', '去除重复词过多的文档。', '1.0.0', 'text', 'text', null, '{"repeatPhraseRatio": {"name": "文档词重复率", "description": "某个词的统计数/文档总词数 > 设定值,该文档被去除。", "type": "slider", "defaultVal": 0.5, "min": 0, "max": 1, "step": 0.1}, "hitStopwords": {"name": "去除停用词", "description": "统计重复词时,选择是否要去除停用词。", "type": "switch", "defaultVal": false, "required": true, "checkedLabel": "去除", "unCheckedLabel": "不去除"}}', '', 'false'), ('FileWithHighRepeatWordRateFilter', '文档字重复率检查', '去除重复字过多的文档。', '1.0.0', 'text', 'text', null, '{"repeatWordRatio": {"name": "文档字重复率", "description": "某个字的统计数/文档总字数 > 设定值,该文档被去除。", "type": "slider", "defaultVal": 0.5, "min": 0, "max": 1, "step": 0.1}}', '', 'false'), ('FileWithHighSpecialCharRateFilter', '文档特殊字符率检查', '去除特殊字符过多的文档。', '1.0.0', 'text', 'text', null, '{"specialCharRatio": {"name": "文档特殊字符率", "description": "特殊字符的统计数/文档总字数 > 设定值,该文档被去除。", "type": "slider", "defaultVal": 0.3, "min": 0, "max": 1, "step": 0.1}}', '', 'false'),