diff --git a/apis/core/v1alpha1/common_types.go b/apis/core/v1alpha1/common_types.go index 66e8dd6a2c..2f2144b379 100644 --- a/apis/core/v1alpha1/common_types.go +++ b/apis/core/v1alpha1/common_types.go @@ -78,6 +78,10 @@ const ( // Users may overlay some fields in managed resource such as pods. Names with this // prefix is preserved to avoid conflicts with fields defined by users. NamePrefix = "ti-" + + // VolNamePrefix is defined for custom persistent volume which may have name conflicts + // with the volumes managed by tidb operator + VolNamePrefix = NamePrefix + "vol-" ) const ( @@ -193,12 +197,8 @@ type Volume struct { // If not specified, the PVC name will be "{component}-{podName}" Name string `json:"name,omitempty"` - // Path is mount path of this volume - Path string `json:"path"` - - // For defines the usage of this volume - // At least one usage is needed for a new volume - For []VolumeUsage `json:"for"` + // Mounts defines mount infos of this volume + Mounts []VolumeMount `json:"mounts"` // Storage defines the request size of this volume Storage resource.Quantity `json:"storage"` @@ -216,15 +216,16 @@ type Volume struct { VolumeAttributesClassName *string `json:"volumeAttributesClassName,omitempty"` } -type VolumeUsage struct { - // Type is a usage type of the volume. - // A volume can be defined for multiple usages. - Type VolumeUsageType `json:"type"` - // SubPath is the relative path of the volume's mount path. +type VolumeMount struct { + // Type is a type of the volume mount. + Type VolumeMountType `json:"type"` + // Mount path of volume, if it's not set, use the default path of this type + MountPath string `json:"mountPath,omitempty"` + // SubPath is the path of the volume's root path. SubPath string `json:"subPath,omitempty"` } -type VolumeUsageType string +type VolumeMountType string // Port defines a listen port type Port struct { diff --git a/apis/core/v1alpha1/pd_types.go b/apis/core/v1alpha1/pd_types.go index 5482a6d902..513ba9feb8 100644 --- a/apis/core/v1alpha1/pd_types.go +++ b/apis/core/v1alpha1/pd_types.go @@ -30,9 +30,11 @@ var ( ) const ( - // VolumeUsageTypePDData means data dir of PD - // The default sub path is "data" - VolumeUsageTypePDData VolumeUsageType = "data" + // VolumeMountTypePDData means data dir of PD + VolumeMountTypePDData VolumeMountType = "data" + + VolumeMountPDDataDefaultPath = "/var/lib/pd" + VolumeMountPDDataDefaultSubPath = "" ) const ( diff --git a/apis/core/v1alpha1/tidb_types.go b/apis/core/v1alpha1/tidb_types.go index ab14807578..12fae31983 100644 --- a/apis/core/v1alpha1/tidb_types.go +++ b/apis/core/v1alpha1/tidb_types.go @@ -30,6 +30,15 @@ var ( _ ComponentAccessor = &TiDB{} ) +const ( + // VolumeUsageTypeTiDBSlowLog means the data dir of slowlog + // The default sub path is "slowlog" + // Users can define a persistent volume for slowlog, or an empty dir will be used. + VolumeMountTypeTiDBSlowLog VolumeMountType = "slowlog" + + VolumeMountTiDBSlowLogDefaultPath = "/var/log/tidb" +) + const ( TiDBPortNameClient = "mysql-client" TiDBPortNameStatus = "status" @@ -81,7 +90,6 @@ const ( const ( TiDBSlowLogContainerName = NamePrefix + "slowlog" TiDBDefaultSlowLogVolumeName = NamePrefix + "slowlog" - TiDBDefaultSlowLogDir = "/var/log/tidb" TiDBSlowLogFileName = "slowlog" ) @@ -406,11 +414,6 @@ type TiDBSlowLog struct { // Default is busybox:1.37.0 Image *string `json:"image,omitempty"` - // VolumeName is the name of the volume used to share the slow log file between the main container and the sidecar. - // If not set, a temparary volume will be used. - // Otherwise, it should be a name of a volume defined in the `volumes` field of the TiDBTemplateSpec. - VolumeName string `json:"volumeName,omitempty"` - // ResourceRequirements defines the resource requirements for the slow log sidecar. Resources ResourceRequirements `json:"resources,omitempty"` } diff --git a/apis/core/v1alpha1/tiflash_types.go b/apis/core/v1alpha1/tiflash_types.go index 41baf0b810..bcdeeed8ac 100644 --- a/apis/core/v1alpha1/tiflash_types.go +++ b/apis/core/v1alpha1/tiflash_types.go @@ -42,9 +42,10 @@ const ( ) const ( - // VolumeUsageTypeTiFlashData is the main data dir for the tiflash - // The default sub path of this type is "" - VolumeUsageTypeTiFlashData VolumeUsageType = "data" + // VolumeMountTypeTiFlashData is the main data dir for the tiflash + VolumeMountTypeTiFlashData VolumeMountType = "data" + + VolumeMountTiFlashDataDefaultPath = "/var/lib/tiflash" ) const ( diff --git a/apis/core/v1alpha1/tikv_types.go b/apis/core/v1alpha1/tikv_types.go index 6bf39a9cd5..a1a20c620e 100644 --- a/apis/core/v1alpha1/tikv_types.go +++ b/apis/core/v1alpha1/tikv_types.go @@ -30,9 +30,11 @@ var ( ) const ( - // VolumeUsageTypeTiKVData is the main data dir for the tikv - // The default sub path of this type is "data" - VolumeUsageTypeTiKVData VolumeUsageType = "data" + // VolumeMountTypeTiKVData is the main data dir for the tikv + // The default sub path of this type is "" + VolumeMountTypeTiKVData VolumeMountType = "data" + + VolumeMountTiKVDataDefaultPath = "/var/lib/tikv" ) const ( diff --git a/apis/core/v1alpha1/zz_generated.deepcopy.go b/apis/core/v1alpha1/zz_generated.deepcopy.go index c04a73bf8a..58eb59d0b3 100644 --- a/apis/core/v1alpha1/zz_generated.deepcopy.go +++ b/apis/core/v1alpha1/zz_generated.deepcopy.go @@ -2012,9 +2012,9 @@ func (in *UpdateStrategy) DeepCopy() *UpdateStrategy { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Volume) DeepCopyInto(out *Volume) { *out = *in - if in.For != nil { - in, out := &in.For, &out.For - *out = make([]VolumeUsage, len(*in)) + if in.Mounts != nil { + in, out := &in.Mounts, &out.Mounts + *out = make([]VolumeMount, len(*in)) copy(*out, *in) } out.Storage = in.Storage.DeepCopy() @@ -2042,17 +2042,17 @@ func (in *Volume) DeepCopy() *Volume { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *VolumeUsage) DeepCopyInto(out *VolumeUsage) { +func (in *VolumeMount) DeepCopyInto(out *VolumeMount) { *out = *in return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeUsage. -func (in *VolumeUsage) DeepCopy() *VolumeUsage { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeMount. +func (in *VolumeMount) DeepCopy() *VolumeMount { if in == nil { return nil } - out := new(VolumeUsage) + out := new(VolumeMount) in.DeepCopyInto(out) return out } diff --git a/manifests/crd/core.pingcap.com_pdgroups.yaml b/manifests/crd/core.pingcap.com_pdgroups.yaml index 76bf63253f..1cc82baa1f 100644 --- a/manifests/crd/core.pingcap.com_pdgroups.yaml +++ b/manifests/crd/core.pingcap.com_pdgroups.yaml @@ -222,20 +222,20 @@ spec: A volume can be mounted for multiple different usages. For example, a volume can be mounted for both data and raft log. properties: - for: - description: |- - For defines the usage of this volume - At least one usage is needed for a new volume + mounts: + description: Mounts defines mount infos of this volume items: properties: + mountPath: + description: Mount path of volume, if it's not + set, use the default path of this type + type: string subPath: - description: SubPath is the relative path of the - volume's mount path. + description: SubPath is the path of the volume's + root path. type: string type: - description: |- - Type is a usage type of the volume. - A volume can be defined for multiple usages. + description: Type is a type of the volume mount. type: string required: - type @@ -246,9 +246,6 @@ spec: Name is volume name. If not specified, the PVC name will be "{component}-{podName}" type: string - path: - description: Path is mount path of this volume - type: string storage: anyOf: - type: integer @@ -272,8 +269,7 @@ spec: It's only available when the feature is enabled. type: string required: - - for - - path + - mounts - storage type: object type: array diff --git a/manifests/crd/core.pingcap.com_pds.yaml b/manifests/crd/core.pingcap.com_pds.yaml index 8fb5afa646..9b97297e6d 100644 --- a/manifests/crd/core.pingcap.com_pds.yaml +++ b/manifests/crd/core.pingcap.com_pds.yaml @@ -161,20 +161,20 @@ spec: A volume can be mounted for multiple different usages. For example, a volume can be mounted for both data and raft log. properties: - for: - description: |- - For defines the usage of this volume - At least one usage is needed for a new volume + mounts: + description: Mounts defines mount infos of this volume items: properties: + mountPath: + description: Mount path of volume, if it's not set, use + the default path of this type + type: string subPath: - description: SubPath is the relative path of the volume's - mount path. + description: SubPath is the path of the volume's root + path. type: string type: - description: |- - Type is a usage type of the volume. - A volume can be defined for multiple usages. + description: Type is a type of the volume mount. type: string required: - type @@ -185,9 +185,6 @@ spec: Name is volume name. If not specified, the PVC name will be "{component}-{podName}" type: string - path: - description: Path is mount path of this volume - type: string storage: anyOf: - type: integer @@ -210,8 +207,7 @@ spec: It's only available when the feature is enabled. type: string required: - - for - - path + - mounts - storage type: object type: array diff --git a/manifests/crd/core.pingcap.com_tidbgroups.yaml b/manifests/crd/core.pingcap.com_tidbgroups.yaml index 6f30d2808f..8c63714ce7 100644 --- a/manifests/crd/core.pingcap.com_tidbgroups.yaml +++ b/manifests/crd/core.pingcap.com_tidbgroups.yaml @@ -335,12 +335,6 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object - volumeName: - description: |- - VolumeName is the name of the volume used to share the slow log file between the main container and the sidecar. - If not set, a temparary volume will be used. - Otherwise, it should be a name of a volume defined in the `volumes` field of the TiDBTemplateSpec. - type: string type: object updateStrategy: properties: @@ -362,20 +356,20 @@ spec: A volume can be mounted for multiple different usages. For example, a volume can be mounted for both data and raft log. properties: - for: - description: |- - For defines the usage of this volume - At least one usage is needed for a new volume + mounts: + description: Mounts defines mount infos of this volume items: properties: + mountPath: + description: Mount path of volume, if it's not + set, use the default path of this type + type: string subPath: - description: SubPath is the relative path of the - volume's mount path. + description: SubPath is the path of the volume's + root path. type: string type: - description: |- - Type is a usage type of the volume. - A volume can be defined for multiple usages. + description: Type is a type of the volume mount. type: string required: - type @@ -386,9 +380,6 @@ spec: Name is volume name. If not specified, the PVC name will be "{component}-{podName}" type: string - path: - description: Path is mount path of this volume - type: string storage: anyOf: - type: integer @@ -412,8 +403,7 @@ spec: It's only available when the feature is enabled. type: string required: - - for - - path + - mounts - storage type: object type: array diff --git a/manifests/crd/core.pingcap.com_tidbs.yaml b/manifests/crd/core.pingcap.com_tidbs.yaml index a875aa2786..aeafd5e204 100644 --- a/manifests/crd/core.pingcap.com_tidbs.yaml +++ b/manifests/crd/core.pingcap.com_tidbs.yaml @@ -233,12 +233,6 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object - volumeName: - description: |- - VolumeName is the name of the volume used to share the slow log file between the main container and the sidecar. - If not set, a temparary volume will be used. - Otherwise, it should be a name of a volume defined in the `volumes` field of the TiDBTemplateSpec. - type: string type: object subdomain: description: |- @@ -276,20 +270,20 @@ spec: A volume can be mounted for multiple different usages. For example, a volume can be mounted for both data and raft log. properties: - for: - description: |- - For defines the usage of this volume - At least one usage is needed for a new volume + mounts: + description: Mounts defines mount infos of this volume items: properties: + mountPath: + description: Mount path of volume, if it's not set, use + the default path of this type + type: string subPath: - description: SubPath is the relative path of the volume's - mount path. + description: SubPath is the path of the volume's root + path. type: string type: - description: |- - Type is a usage type of the volume. - A volume can be defined for multiple usages. + description: Type is a type of the volume mount. type: string required: - type @@ -300,9 +294,6 @@ spec: Name is volume name. If not specified, the PVC name will be "{component}-{podName}" type: string - path: - description: Path is mount path of this volume - type: string storage: anyOf: - type: integer @@ -325,8 +316,7 @@ spec: It's only available when the feature is enabled. type: string required: - - for - - path + - mounts - storage type: object type: array diff --git a/manifests/crd/core.pingcap.com_tiflashes.yaml b/manifests/crd/core.pingcap.com_tiflashes.yaml index 637779fb96..143188fcfb 100644 --- a/manifests/crd/core.pingcap.com_tiflashes.yaml +++ b/manifests/crd/core.pingcap.com_tiflashes.yaml @@ -205,20 +205,20 @@ spec: A volume can be mounted for multiple different usages. For example, a volume can be mounted for both data and raft log. properties: - for: - description: |- - For defines the usage of this volume - At least one usage is needed for a new volume + mounts: + description: Mounts defines mount infos of this volume items: properties: + mountPath: + description: Mount path of volume, if it's not set, use + the default path of this type + type: string subPath: - description: SubPath is the relative path of the volume's - mount path. + description: SubPath is the path of the volume's root + path. type: string type: - description: |- - Type is a usage type of the volume. - A volume can be defined for multiple usages. + description: Type is a type of the volume mount. type: string required: - type @@ -229,9 +229,6 @@ spec: Name is volume name. If not specified, the PVC name will be "{component}-{podName}" type: string - path: - description: Path is mount path of this volume - type: string storage: anyOf: - type: integer @@ -254,8 +251,7 @@ spec: It's only available when the feature is enabled. type: string required: - - for - - path + - mounts - storage type: object type: array diff --git a/manifests/crd/core.pingcap.com_tiflashgroups.yaml b/manifests/crd/core.pingcap.com_tiflashgroups.yaml index afd0add86e..609ed40703 100644 --- a/manifests/crd/core.pingcap.com_tiflashgroups.yaml +++ b/manifests/crd/core.pingcap.com_tiflashgroups.yaml @@ -258,20 +258,20 @@ spec: A volume can be mounted for multiple different usages. For example, a volume can be mounted for both data and raft log. properties: - for: - description: |- - For defines the usage of this volume - At least one usage is needed for a new volume + mounts: + description: Mounts defines mount infos of this volume items: properties: + mountPath: + description: Mount path of volume, if it's not + set, use the default path of this type + type: string subPath: - description: SubPath is the relative path of the - volume's mount path. + description: SubPath is the path of the volume's + root path. type: string type: - description: |- - Type is a usage type of the volume. - A volume can be defined for multiple usages. + description: Type is a type of the volume mount. type: string required: - type @@ -282,9 +282,6 @@ spec: Name is volume name. If not specified, the PVC name will be "{component}-{podName}" type: string - path: - description: Path is mount path of this volume - type: string storage: anyOf: - type: integer @@ -308,8 +305,7 @@ spec: It's only available when the feature is enabled. type: string required: - - for - - path + - mounts - storage type: object type: array diff --git a/manifests/crd/core.pingcap.com_tikvgroups.yaml b/manifests/crd/core.pingcap.com_tikvgroups.yaml index 5bb8cb471e..e41751bff1 100644 --- a/manifests/crd/core.pingcap.com_tikvgroups.yaml +++ b/manifests/crd/core.pingcap.com_tikvgroups.yaml @@ -223,20 +223,20 @@ spec: A volume can be mounted for multiple different usages. For example, a volume can be mounted for both data and raft log. properties: - for: - description: |- - For defines the usage of this volume - At least one usage is needed for a new volume + mounts: + description: Mounts defines mount infos of this volume items: properties: + mountPath: + description: Mount path of volume, if it's not + set, use the default path of this type + type: string subPath: - description: SubPath is the relative path of the - volume's mount path. + description: SubPath is the path of the volume's + root path. type: string type: - description: |- - Type is a usage type of the volume. - A volume can be defined for multiple usages. + description: Type is a type of the volume mount. type: string required: - type @@ -247,9 +247,6 @@ spec: Name is volume name. If not specified, the PVC name will be "{component}-{podName}" type: string - path: - description: Path is mount path of this volume - type: string storage: anyOf: - type: integer @@ -273,8 +270,7 @@ spec: It's only available when the feature is enabled. type: string required: - - for - - path + - mounts - storage type: object type: array diff --git a/manifests/crd/core.pingcap.com_tikvs.yaml b/manifests/crd/core.pingcap.com_tikvs.yaml index 5eebc54316..ef6d77206d 100644 --- a/manifests/crd/core.pingcap.com_tikvs.yaml +++ b/manifests/crd/core.pingcap.com_tikvs.yaml @@ -165,20 +165,20 @@ spec: A volume can be mounted for multiple different usages. For example, a volume can be mounted for both data and raft log. properties: - for: - description: |- - For defines the usage of this volume - At least one usage is needed for a new volume + mounts: + description: Mounts defines mount infos of this volume items: properties: + mountPath: + description: Mount path of volume, if it's not set, use + the default path of this type + type: string subPath: - description: SubPath is the relative path of the volume's - mount path. + description: SubPath is the path of the volume's root + path. type: string type: - description: |- - Type is a usage type of the volume. - A volume can be defined for multiple usages. + description: Type is a type of the volume mount. type: string required: - type @@ -189,9 +189,6 @@ spec: Name is volume name. If not specified, the PVC name will be "{component}-{podName}" type: string - path: - description: Path is mount path of this volume - type: string storage: anyOf: - type: integer @@ -214,8 +211,7 @@ spec: It's only available when the feature is enabled. type: string required: - - for - - path + - mounts - storage type: object type: array diff --git a/pkg/configs/pd/config.go b/pkg/configs/pd/config.go index a4c42e8262..edf2fa154b 100644 --- a/pkg/configs/pd/config.go +++ b/pkg/configs/pd/config.go @@ -79,16 +79,18 @@ func (c *Config) Overlay(cluster *v1alpha1.Cluster, pd *v1alpha1.PD, peers []*v1 for i := range pd.Spec.Volumes { vol := &pd.Spec.Volumes[i] - for _, usage := range vol.For { - if usage.Type == v1alpha1.VolumeUsageTypePDData { - c.DataDir = vol.Path - if usage.SubPath != "" { - c.DataDir = path.Join(vol.Path, usage.SubPath) - } + for k := range vol.Mounts { + mount := &vol.Mounts[k] + if mount.Type == v1alpha1.VolumeMountTypePDData { + c.DataDir = mount.MountPath } } } + if c.DataDir == "" { + c.DataDir = v1alpha1.VolumeMountPDDataDefaultPath + } + c.InitialClusterToken = pd.Spec.Cluster.Name initialClusterNum, ok := pd.Annotations[v1alpha1.AnnoKeyInitialClusterNum] if !ok { diff --git a/pkg/configs/pd/config_test.go b/pkg/configs/pd/config_test.go index d62caf17dc..58c1574d24 100644 --- a/pkg/configs/pd/config_test.go +++ b/pkg/configs/pd/config_test.go @@ -88,10 +88,9 @@ func TestOverlay(t *testing.T) { Volumes: []v1alpha1.Volume{ { Name: "data", - Path: "/var/lib/pd", - For: []v1alpha1.VolumeUsage{ + Mounts: []v1alpha1.VolumeMount{ { - Type: v1alpha1.VolumeUsageTypePDData, + Type: v1alpha1.VolumeMountTypePDData, }, }, Storage: resource.Quantity{ diff --git a/pkg/configs/tidb/config.go b/pkg/configs/tidb/config.go index ccd2e6cb30..9258ac5e3c 100644 --- a/pkg/configs/tidb/config.go +++ b/pkg/configs/tidb/config.go @@ -179,16 +179,25 @@ func removeHTTPPrefix(url string) string { func getSlowQueryFile(tidb *v1alpha1.TiDB) string { if !tidb.IsSeparateSlowLogEnabled() { return "" - } else if tidb.Spec.SlowLog == nil || tidb.Spec.SlowLog.VolumeName == "" { - return path.Join(v1alpha1.TiDBDefaultSlowLogDir, v1alpha1.TiDBSlowLogFileName) } + var dir string for i := range tidb.Spec.Volumes { vol := &tidb.Spec.Volumes[i] - if vol.Name == tidb.Spec.SlowLog.VolumeName { - return path.Join(vol.Path, v1alpha1.TiDBSlowLogFileName) + for k := range vol.Mounts { + mount := &vol.Mounts[k] + + if mount.Type != v1alpha1.VolumeMountTypeTiDBSlowLog { + continue + } + + dir = mount.MountPath } } - return "" // should not reach here + if dir == "" { + dir = v1alpha1.VolumeMountTiDBSlowLogDefaultPath + } + + return path.Join(dir, v1alpha1.TiDBSlowLogFileName) } diff --git a/pkg/configs/tidb/config_test.go b/pkg/configs/tidb/config_test.go index 6fab1c0948..ae36616d05 100644 --- a/pkg/configs/tidb/config_test.go +++ b/pkg/configs/tidb/config_test.go @@ -130,17 +130,18 @@ func TestOverlay(t *testing.T) { // store slowlog in PVC tidb2 := tidb.DeepCopy() - tidb2.Spec.SlowLog = &v1alpha1.TiDBSlowLog{ - VolumeName: "slowlog", - } tidb2.Spec.Volumes = []v1alpha1.Volume{ { Name: "slowlog", - Path: "/var/log/slowlog", + Mounts: []v1alpha1.VolumeMount{ + { + Type: v1alpha1.VolumeMountTypeTiDBSlowLog, + }, + }, }, } cfg2 := &Config{} err = cfg2.Overlay(cluster, tidb2) require.NoError(t, err) - assert.Equal(t, "/var/log/slowlog/slowlog", cfg2.Log.SlowQueryFile) + assert.Equal(t, "/var/log/tidb/slowlog", cfg2.Log.SlowQueryFile) } diff --git a/pkg/configs/tiflash/config.go b/pkg/configs/tiflash/config.go index 5d1ff09293..d75ee1b20b 100644 --- a/pkg/configs/tiflash/config.go +++ b/pkg/configs/tiflash/config.go @@ -102,26 +102,24 @@ func (c *Config) Overlay(cluster *v1alpha1.Cluster, tiflash *v1alpha1.TiFlash) e c.Security.KeyPath = path.Join(v1alpha1.TiFlashClusterTLSMountPath, corev1.TLSPrivateKeyKey) } + var dataDir string for i := range tiflash.Spec.Volumes { vol := &tiflash.Spec.Volumes[i] - for _, usage := range vol.For { - if usage.Type != v1alpha1.VolumeUsageTypeTiFlashData { + for _, mount := range vol.Mounts { + if mount.Type != v1alpha1.VolumeMountTypeTiFlashData { continue } - dataDir := vol.Path - if usage.SubPath != "" { - dataDir = path.Join(vol.Path, usage.SubPath) - } - - c.TmpPath = getTmpPath(dataDir) - c.Storage.Main.Dir = []string{getMainStorageDir(dataDir)} - c.Storage.Raft.Dir = []string{getRaftStorageDir(dataDir)} - c.Flash.Proxy.DataDir = getProxyDataDir(dataDir) - c.Logger.Log = GetServerLogPath(dataDir) - c.Logger.Errorlog = GetErrorLogPath(dataDir) + dataDir = mount.MountPath } } + c.TmpPath = getTmpPath(dataDir) + c.Storage.Main.Dir = []string{getMainStorageDir(dataDir)} + c.Storage.Raft.Dir = []string{getRaftStorageDir(dataDir)} + c.Flash.Proxy.DataDir = getProxyDataDir(dataDir) + c.Logger.Log = GetServerLogPath(dataDir) + c.Logger.Errorlog = GetErrorLogPath(dataDir) + c.Flash.ServiceAddr = GetServiceAddr(tiflash) // /etc/tiflash/proxy.toml c.Flash.Proxy.Config = path.Join(v1alpha1.DirNameConfigTiFlash, v1alpha1.ConfigFileTiFlashProxyName) @@ -229,26 +227,40 @@ func getProxyAdvertiseStatusAddr(tiflash *v1alpha1.TiFlash) string { return fmt.Sprintf("%s.%s.%s:%d", tiflash.PodName(), tiflash.Spec.Subdomain, ns, tiflash.GetProxyStatusPort()) } +func GetDataDir(dataDir string) string { + if dataDir == "" { + return v1alpha1.VolumeMountTiFlashDataDefaultPath + } + + return dataDir +} + func GetServerLogPath(dataDir string) string { + dataDir = GetDataDir(dataDir) return fmt.Sprintf("%s/logs/server.log", dataDir) } func GetErrorLogPath(dataDir string) string { + dataDir = GetDataDir(dataDir) return fmt.Sprintf("%s/logs/error.log", dataDir) } func getTmpPath(dataDir string) string { + dataDir = GetDataDir(dataDir) return fmt.Sprintf("%s/tmp", dataDir) } func getMainStorageDir(dataDir string) string { + dataDir = GetDataDir(dataDir) return fmt.Sprintf("%s/db", dataDir) } func getRaftStorageDir(dataDir string) string { + dataDir = GetDataDir(dataDir) return fmt.Sprintf("%s/kvstore", dataDir) } func getProxyDataDir(dataDir string) string { + dataDir = GetDataDir(dataDir) return fmt.Sprintf("%s/proxy", dataDir) } diff --git a/pkg/configs/tiflash/config_test.go b/pkg/configs/tiflash/config_test.go index 1982d50c4e..59ac419a85 100644 --- a/pkg/configs/tiflash/config_test.go +++ b/pkg/configs/tiflash/config_test.go @@ -109,10 +109,10 @@ func TestOverlay(t *testing.T) { Volumes: []v1alpha1.Volume{ { Name: "data", - Path: "/data0", - For: []v1alpha1.VolumeUsage{ + Mounts: []v1alpha1.VolumeMount{ { - Type: v1alpha1.VolumeUsageTypeTiFlashData, + MountPath: "/data0", + Type: v1alpha1.VolumeMountTypeTiFlashData, }, }, }, diff --git a/pkg/configs/tikv/config.go b/pkg/configs/tikv/config.go index 6106a74452..0cf60f5164 100644 --- a/pkg/configs/tikv/config.go +++ b/pkg/configs/tikv/config.go @@ -73,16 +73,17 @@ func (c *Config) Overlay(cluster *v1alpha1.Cluster, tikv *v1alpha1.TiKV) error { for i := range tikv.Spec.Volumes { vol := &tikv.Spec.Volumes[i] - for _, usage := range vol.For { - if usage.Type == v1alpha1.VolumeUsageTypeTiKVData { - c.Storage.DataDir = vol.Path - if usage.SubPath != "" { - c.Storage.DataDir = path.Join(vol.Path, usage.SubPath) - } + for _, mount := range vol.Mounts { + if mount.Type == v1alpha1.VolumeMountTypeTiKVData { + c.Storage.DataDir = mount.MountPath } } } + if c.Storage.DataDir == "" { + c.Storage.DataDir = v1alpha1.VolumeMountTiKVDataDefaultPath + } + return nil } diff --git a/pkg/configs/tikv/config_test.go b/pkg/configs/tikv/config_test.go index 62543d1b9f..0a693fa1c6 100644 --- a/pkg/configs/tikv/config_test.go +++ b/pkg/configs/tikv/config_test.go @@ -86,10 +86,9 @@ func TestOverlay(t *testing.T) { Volumes: []v1alpha1.Volume{ { Name: "data", - Path: "/var/lib/tikv", - For: []v1alpha1.VolumeUsage{ + Mounts: []v1alpha1.VolumeMount{ { - Type: v1alpha1.VolumeUsageTypeTiKVData, + Type: v1alpha1.VolumeMountTypeTiKVData, }, }, Storage: resource.Quantity{ diff --git a/pkg/controllers/pd/tasks/pod.go b/pkg/controllers/pd/tasks/pod.go index 2a742e8247..9503b1a964 100644 --- a/pkg/controllers/pd/tasks/pod.go +++ b/pkg/controllers/pd/tasks/pod.go @@ -151,10 +151,7 @@ func newPod(cluster *v1alpha1.Cluster, pd *v1alpha1.PD, configHash string) *core for i := range pd.Spec.Volumes { vol := &pd.Spec.Volumes[i] - name := v1alpha1.NamePrefix + "pd" - if vol.Name != "" { - name = name + "-" + vol.Name - } + name := VolumeName(vol.Name) vols = append(vols, corev1.Volume{ Name: name, VolumeSource: corev1.VolumeSource{ @@ -163,10 +160,10 @@ func newPod(cluster *v1alpha1.Cluster, pd *v1alpha1.PD, configHash string) *core }, }, }) - mounts = append(mounts, corev1.VolumeMount{ - Name: name, - MountPath: vol.Path, - }) + for i := range vol.Mounts { + mount := VolumeMount(name, &vol.Mounts[i]) + mounts = append(mounts, *mount) + } } if cluster.IsTLSClusterEnabled() { diff --git a/pkg/controllers/pd/tasks/util.go b/pkg/controllers/pd/tasks/util.go index 51f5602f39..2cd76d6188 100644 --- a/pkg/controllers/pd/tasks/util.go +++ b/pkg/controllers/pd/tasks/util.go @@ -17,6 +17,7 @@ package tasks import ( "time" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -48,3 +49,25 @@ func LongestHealthPeer(pd *v1alpha1.PD, peers []*v1alpha1.PD) string { return p } + +// Real spec.volumes[*].name of pod +// TODO(liubo02): extract to namer pkg +func VolumeName(volName string) string { + return v1alpha1.VolNamePrefix + volName +} + +func VolumeMount(name string, mount *v1alpha1.VolumeMount) *corev1.VolumeMount { + vm := &corev1.VolumeMount{ + Name: name, + MountPath: mount.MountPath, + SubPath: mount.SubPath, + } + + if mount.Type == v1alpha1.VolumeMountTypePDData { + if vm.MountPath == "" { + vm.MountPath = v1alpha1.VolumeMountPDDataDefaultPath + } + } + + return vm +} diff --git a/pkg/controllers/tidb/tasks/ctx.go b/pkg/controllers/tidb/tasks/ctx.go index a2161168ac..32b2befebe 100644 --- a/pkg/controllers/tidb/tasks/ctx.go +++ b/pkg/controllers/tidb/tasks/ctx.go @@ -40,8 +40,7 @@ type ReconcileContext struct { TiDBClient tidbapi.TiDBClient PDClient pdapi.PDClient - Healthy bool - Suspended bool + Healthy bool GracefulWaitTimeInSeconds int64 diff --git a/pkg/controllers/tidb/tasks/pod.go b/pkg/controllers/tidb/tasks/pod.go index 1f009e7342..16a00e04bc 100644 --- a/pkg/controllers/tidb/tasks/pod.go +++ b/pkg/controllers/tidb/tasks/pod.go @@ -110,9 +110,10 @@ func newPod(cluster *v1alpha1.Cluster, }, } + var slowLogMount *corev1.VolumeMount for i := range tidb.Spec.Volumes { vol := &tidb.Spec.Volumes[i] - name := genVolumeNameFromVolume(vol) + name := VolumeName(vol.Name) vols = append(vols, corev1.Volume{ Name: name, VolumeSource: corev1.VolumeSource{ @@ -121,10 +122,15 @@ func newPod(cluster *v1alpha1.Cluster, }, }, }) - mounts = append(mounts, corev1.VolumeMount{ - Name: name, - MountPath: vol.Path, - }) + + for i := range vol.Mounts { + mount := &vol.Mounts[i] + vm := VolumeMount(name, mount) + mounts = append(mounts, *vm) + if mount.Type == v1alpha1.VolumeMountTypeTiDBSlowLog { + slowLogMount = vm + } + } } if tidb.IsMySQLTLSEnabled() { @@ -201,14 +207,14 @@ func newPod(cluster *v1alpha1.Cluster, var slowLogContainer *corev1.Container if tidb.IsSeparateSlowLogEnabled() { - vol, mount := buildSlowLogVolumeAndMount(tidb) - if vol != nil { + // no persistent slowlog volume + if slowLogMount == nil { + vol, mount := defaultSlowLogVolumeAndMount() vols = append(vols, *vol) - } - if mount != nil { mounts = append(mounts, *mount) + slowLogMount = mount } - slowLogContainer = buildSlowLogContainer(tidb, mount) + slowLogContainer = buildSlowLogContainer(tidb, slowLogMount) } pod := &corev1.Pod{ @@ -284,13 +290,7 @@ func newPod(cluster *v1alpha1.Cluster, return pod } -func genVolumeNameFromVolume(vol *v1alpha1.Volume) string { - name := v1alpha1.NamePrefix + "tidb" - if vol.Name != "" { - name = name + "-" + vol.Name - } - return name -} +// TODO(liubo02): extract to namer pkg func buildTiDBReadinessProbHandler(cluster *v1alpha1.Cluster, tidb *v1alpha1.TiDB, clientPort, statusPort int32) corev1.ProbeHandler { probeType := v1alpha1.TCPProbeType // default to TCP probe @@ -338,38 +338,19 @@ func buildTiDBProbeCommand(cluster *v1alpha1.Cluster, statusPort int32) (command return } -func buildSlowLogVolumeAndMount(tidb *v1alpha1.TiDB) (*corev1.Volume, *corev1.VolumeMount) { - if tidb.Spec.SlowLog == nil || tidb.Spec.SlowLog.VolumeName == "" { - return &corev1.Volume{ - Name: v1alpha1.TiDBDefaultSlowLogVolumeName, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, &corev1.VolumeMount{ - Name: v1alpha1.TiDBDefaultSlowLogVolumeName, - MountPath: v1alpha1.TiDBDefaultSlowLogDir, - } - } - - // if using a custom volume, the volume and mount should already be defined - return nil, nil +func defaultSlowLogVolumeAndMount() (*corev1.Volume, *corev1.VolumeMount) { + return &corev1.Volume{ + Name: v1alpha1.TiDBDefaultSlowLogVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, &corev1.VolumeMount{ + Name: v1alpha1.TiDBDefaultSlowLogVolumeName, + MountPath: v1alpha1.VolumeMountTiDBSlowLogDefaultPath, + } } func buildSlowLogContainer(tidb *v1alpha1.TiDB, mount *corev1.VolumeMount) *corev1.Container { - if mount == nil { - // no temparary volume for slow log, find the volume defined in the spec - for i := range tidb.Spec.Volumes { - vol := &tidb.Spec.Volumes[i] - if vol.Name == tidb.Spec.SlowLog.VolumeName { - mount = &corev1.VolumeMount{ - Name: genVolumeNameFromVolume(vol), - MountPath: vol.Path, - } - break // should always find the volume - } - } - } - slowlogFile := path.Join(mount.MountPath, v1alpha1.TiDBSlowLogFileName) img := v1alpha1.DefaultHelperImage if tidb.Spec.SlowLog != nil && tidb.Spec.SlowLog.Image != nil && *tidb.Spec.SlowLog.Image != "" { diff --git a/pkg/controllers/tidb/tasks/util.go b/pkg/controllers/tidb/tasks/util.go index 3fc4502fb6..2859bd922f 100644 --- a/pkg/controllers/tidb/tasks/util.go +++ b/pkg/controllers/tidb/tasks/util.go @@ -17,6 +17,8 @@ package tasks import ( "fmt" + corev1 "k8s.io/api/core/v1" + "github.com/pingcap/tidb-operator/apis/core/v1alpha1" ) @@ -34,3 +36,24 @@ func PersistentVolumeClaimName(podName, volName string) string { func TiDBServiceURL(tidb *v1alpha1.TiDB, scheme string) string { return fmt.Sprintf("%s://%s.%s.%s.svc:%d", scheme, tidb.PodName(), tidb.Spec.Subdomain, tidb.Namespace, tidb.GetStatusPort()) } + +// Real spec.volumes[*].name of pod +// TODO(liubo02): extract to namer pkg +func VolumeName(volName string) string { + return v1alpha1.VolNamePrefix + volName +} + +func VolumeMount(name string, mount *v1alpha1.VolumeMount) *corev1.VolumeMount { + vm := &corev1.VolumeMount{ + Name: name, + MountPath: mount.MountPath, + SubPath: mount.SubPath, + } + if mount.Type == v1alpha1.VolumeMountTypeTiDBSlowLog { + if vm.MountPath == "" { + vm.MountPath = v1alpha1.VolumeMountTiDBSlowLogDefaultPath + } + } + + return vm +} diff --git a/pkg/controllers/tiflash/tasks/pod.go b/pkg/controllers/tiflash/tasks/pod.go index 65b563bbfe..ddd8d92bb7 100644 --- a/pkg/controllers/tiflash/tasks/pod.go +++ b/pkg/controllers/tiflash/tasks/pod.go @@ -17,7 +17,6 @@ package tasks import ( "context" "fmt" - "path" "path/filepath" corev1 "k8s.io/api/core/v1" @@ -99,7 +98,7 @@ func newPod(cluster *v1alpha1.Cluster, tiflash *v1alpha1.TiFlash, configHash str var dataDir string for i := range tiflash.Spec.Volumes { vol := &tiflash.Spec.Volumes[i] - name := v1alpha1.NamePrefix + "-" + vol.Name + name := VolumeName(vol.Name) vols = append(vols, corev1.Volume{ Name: name, VolumeSource: corev1.VolumeSource{ @@ -108,19 +107,14 @@ func newPod(cluster *v1alpha1.Cluster, tiflash *v1alpha1.TiFlash, configHash str }, }, }) - mount := corev1.VolumeMount{ - Name: name, - MountPath: vol.Path, - } - mounts = append(mounts, mount) - for _, usage := range vol.For { - if usage.Type == v1alpha1.VolumeUsageTypeTiFlashData { - dataMount = &mount - dataDir = vol.Path - if usage.SubPath != "" { - dataDir = path.Join(vol.Path, usage.SubPath) - } + for i := range vol.Mounts { + mount := &vol.Mounts[i] + vm := VolumeMount(name, mount) + if mount.Type == v1alpha1.VolumeMountTypeTiFlashData { + dataMount = vm + dataDir = dataMount.MountPath } + mounts = append(mounts, *vm) } } diff --git a/pkg/controllers/tiflash/tasks/pod_test.go b/pkg/controllers/tiflash/tasks/pod_test.go index 5212716c97..75ddc37ac7 100644 --- a/pkg/controllers/tiflash/tasks/pod_test.go +++ b/pkg/controllers/tiflash/tasks/pod_test.go @@ -30,7 +30,10 @@ import ( "github.com/pingcap/tidb-operator/pkg/utils/task/v3" ) -const fakeVersion = "v1.2.3" +const ( + fakeVersion = "v1.2.3" + fakePodHash = "779dbdbdf4" +) func TestTaskPod(t *testing.T) { cases := []struct { @@ -123,7 +126,7 @@ func TestTaskPod(t *testing.T) { pod: fake.FakeObj("aaa-tiflash-xxx", func(obj *corev1.Pod) *corev1.Pod { obj.Labels = map[string]string{ v1alpha1.LabelKeyConfigHash: "newest", - v1alpha1.LabelKeyPodSpecHash: "556796549d", + v1alpha1.LabelKeyPodSpecHash: fakePodHash, } return obj }), @@ -147,7 +150,7 @@ func TestTaskPod(t *testing.T) { pod: fake.FakeObj("aaa-tiflash-xxx", func(obj *corev1.Pod) *corev1.Pod { obj.Labels = map[string]string{ v1alpha1.LabelKeyConfigHash: "old", - v1alpha1.LabelKeyPodSpecHash: "556796549d", + v1alpha1.LabelKeyPodSpecHash: fakePodHash, } return obj }), @@ -171,7 +174,7 @@ func TestTaskPod(t *testing.T) { pod: fake.FakeObj("aaa-tiflash-xxx", func(obj *corev1.Pod) *corev1.Pod { obj.Labels = map[string]string{ v1alpha1.LabelKeyConfigHash: "newest", - v1alpha1.LabelKeyPodSpecHash: "556796549d", + v1alpha1.LabelKeyPodSpecHash: fakePodHash, "xxx": "yyy", } return obj @@ -196,7 +199,7 @@ func TestTaskPod(t *testing.T) { pod: fake.FakeObj("aaa-tiflash-xxx", func(obj *corev1.Pod) *corev1.Pod { obj.Labels = map[string]string{ v1alpha1.LabelKeyConfigHash: "newest", - v1alpha1.LabelKeyPodSpecHash: "556796549d", + v1alpha1.LabelKeyPodSpecHash: fakePodHash, "xxx": "yyy", } return obj @@ -222,7 +225,7 @@ func TestTaskPod(t *testing.T) { obj.Labels = map[string]string{ v1alpha1.LabelKeyInstance: "aaa-xxx", v1alpha1.LabelKeyConfigHash: "newest", - v1alpha1.LabelKeyPodSpecHash: "556796549d", + v1alpha1.LabelKeyPodSpecHash: fakePodHash, } return obj }), diff --git a/pkg/controllers/tiflash/tasks/util.go b/pkg/controllers/tiflash/tasks/util.go index 7e159c12ac..0997afa188 100644 --- a/pkg/controllers/tiflash/tasks/util.go +++ b/pkg/controllers/tiflash/tasks/util.go @@ -16,6 +16,10 @@ package tasks import ( "fmt" + + corev1 "k8s.io/api/core/v1" + + "github.com/pingcap/tidb-operator/apis/core/v1alpha1" ) func ConfigMapName(podName string) string { @@ -27,3 +31,24 @@ func PersistentVolumeClaimName(podName, volName string) string { // NOTE: for v1, volName should be data0, data1, ... return fmt.Sprintf("%s-%s", volName, podName) } + +// Real spec.volumes[*].name of pod +// TODO(liubo02): extract to namer pkg +func VolumeName(volName string) string { + return v1alpha1.VolNamePrefix + volName +} + +func VolumeMount(name string, mount *v1alpha1.VolumeMount) *corev1.VolumeMount { + vm := &corev1.VolumeMount{ + Name: name, + MountPath: mount.MountPath, + SubPath: mount.SubPath, + } + if mount.Type == v1alpha1.VolumeMountTypeTiFlashData { + if vm.MountPath == "" { + vm.MountPath = v1alpha1.VolumeMountTiFlashDataDefaultPath + } + } + + return vm +} diff --git a/pkg/controllers/tikv/tasks/pod.go b/pkg/controllers/tikv/tasks/pod.go index 5d6af8de58..386e9d2ca6 100644 --- a/pkg/controllers/tikv/tasks/pod.go +++ b/pkg/controllers/tikv/tasks/pod.go @@ -151,10 +151,7 @@ func newPod(cluster *v1alpha1.Cluster, tikv *v1alpha1.TiKV, configHash string) * for i := range tikv.Spec.Volumes { vol := &tikv.Spec.Volumes[i] - name := v1alpha1.NamePrefix + "tikv" - if vol.Name != "" { - name = name + "-" + vol.Name - } + name := VolumeName(vol.Name) vols = append(vols, corev1.Volume{ Name: name, VolumeSource: corev1.VolumeSource{ @@ -163,10 +160,10 @@ func newPod(cluster *v1alpha1.Cluster, tikv *v1alpha1.TiKV, configHash string) * }, }, }) - mounts = append(mounts, corev1.VolumeMount{ - Name: name, - MountPath: vol.Path, - }) + for i := range vol.Mounts { + mount := VolumeMount(name, &vol.Mounts[i]) + mounts = append(mounts, *mount) + } } if cluster.IsTLSClusterEnabled() { diff --git a/pkg/controllers/tikv/tasks/util.go b/pkg/controllers/tikv/tasks/util.go index d85f8cedca..a0d5c95831 100644 --- a/pkg/controllers/tikv/tasks/util.go +++ b/pkg/controllers/tikv/tasks/util.go @@ -20,6 +20,7 @@ import ( corev1 "k8s.io/api/core/v1" + "github.com/pingcap/tidb-operator/apis/core/v1alpha1" "github.com/pingcap/tidb-operator/pkg/client" ) @@ -57,3 +58,24 @@ func CalcGracePeriod(regionCount int) int64 { return gracePeriod } + +// Real spec.volumes[*].name of pod +// TODO(liubo02): extract to namer pkg +func VolumeName(volName string) string { + return v1alpha1.VolNamePrefix + volName +} + +func VolumeMount(name string, mount *v1alpha1.VolumeMount) *corev1.VolumeMount { + vm := &corev1.VolumeMount{ + Name: name, + MountPath: mount.MountPath, + SubPath: mount.SubPath, + } + if mount.Type == v1alpha1.VolumeMountTypeTiKVData { + if vm.MountPath == "" { + vm.MountPath = v1alpha1.VolumeMountTiKVDataDefaultPath + } + } + + return vm +} diff --git a/tests/e2e/data/pd.go b/tests/e2e/data/pd.go index af335e250a..712eefb97d 100644 --- a/tests/e2e/data/pd.go +++ b/tests/e2e/data/pd.go @@ -36,8 +36,7 @@ func NewPDGroup(ns string, patches ...GroupPatch[*runtime.PDGroup]) *v1alpha1.PD Volumes: []v1alpha1.Volume{ { Name: "data", - Path: "/var/lib/pd", - For: []v1alpha1.VolumeUsage{{Type: "data"}}, + Mounts: []v1alpha1.VolumeMount{{Type: "data"}}, Storage: resource.MustParse("1Gi"), }, }, diff --git a/tests/e2e/utils/data/cluster.go b/tests/e2e/utils/data/cluster.go index 1621832161..7d1a4287f4 100644 --- a/tests/e2e/utils/data/cluster.go +++ b/tests/e2e/utils/data/cluster.go @@ -68,8 +68,7 @@ func NewPDGroup(namespace, name, clusterName string, replicas *int32, apply func Volumes: []v1alpha1.Volume{ { Name: "data", - Path: "/var/lib/pd", - For: []v1alpha1.VolumeUsage{{Type: "data"}}, + Mounts: []v1alpha1.VolumeMount{{Type: "data"}}, Storage: StorageSizeGi2quantity(1), }, }, @@ -104,8 +103,7 @@ func NewTiKVGroup(namespace, name, clusterName string, replicas *int32, apply fu Volumes: []v1alpha1.Volume{ { Name: "data", - Path: "/var/lib/tikv", - For: []v1alpha1.VolumeUsage{{Type: "data"}}, + Mounts: []v1alpha1.VolumeMount{{Type: "data"}}, Storage: StorageSizeGi2quantity(1), }, }, @@ -151,7 +149,8 @@ func NewTiDBGroup(namespace, name, clusterName string, replicas *int32, apply fu } func NewTiFlashGroup(namespace, name, clusterName string, replicas *int32, - apply func(group *v1alpha1.TiFlashGroup)) *v1alpha1.TiFlashGroup { + apply func(group *v1alpha1.TiFlashGroup), +) *v1alpha1.TiFlashGroup { flashg := &v1alpha1.TiFlashGroup{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, @@ -172,8 +171,7 @@ func NewTiFlashGroup(namespace, name, clusterName string, replicas *int32, Volumes: []v1alpha1.Volume{ { Name: "data", - Path: "/data0", // different path format - For: []v1alpha1.VolumeUsage{{Type: "data"}}, + Mounts: []v1alpha1.VolumeMount{{Type: "data"}}, Storage: StorageSizeGi2quantity(1), }, },