chore: bump kubeblocks to v1.1.x

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/Masterminds/semver/v3 v3.3.0
 	github.com/NimbleMarkets/ntcharts v0.1.2
 	github.com/apecloud/dbctl v0.0.0-20240827084000-68a1980b1a46
-	github.com/apecloud/kubeblocks v1.0.3-beta.8
+	github.com/apecloud/kubeblocks v1.1.0-beta.5
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
 	github.com/briandowns/spinner v1.23.0
 	github.com/chaos-mesh/chaos-mesh/api v0.0.0-20230912020346-a5d89c1c90ad

go.sum

@@ -695,8 +695,8 @@ github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4x
 github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
 github.com/apecloud/dbctl v0.0.0-20240827084000-68a1980b1a46 h1:+Jcc7IjDGxPgIfIkGX2Q5Yxj35U65zgcfjh0B9rDhjo=
 github.com/apecloud/dbctl v0.0.0-20240827084000-68a1980b1a46/go.mod h1:eksJtZ7z1nVcVLqDzAdcN5EfpHwXllIAvHZEks2zWys=
-github.com/apecloud/kubeblocks v1.0.3-beta.8 h1:9HPgzrLg537dof599VyNRX/gjEFK9Kb819ofDJxz9c0=
-github.com/apecloud/kubeblocks v1.0.3-beta.8/go.mod h1:lB4LCrNPjHlgkDrBr1jTFVepwGUEU6wjZ7jMeCpM0V4=
+github.com/apecloud/kubeblocks v1.1.0-beta.5 h1:a3Jng19OOquH12SqqEBi/pd6fMAeD01NCInLZe9nTP4=
+github.com/apecloud/kubeblocks v1.1.0-beta.5/go.mod h1:vx9p2tv5x4NZiShUvl1MXVl9RO343Bu1nCpbAKdcA6k=
 github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
 github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=

pkg/cluster/cluster.go

@@ -310,7 +310,7 @@ func (o *ClusterObjects) GetComponentInfo() []*ComponentInfo {
 	var comps []*ComponentInfo
 	setComponentInfos := func(compSpec kbappsv1.ClusterComponentSpec,
 		resources corev1.ResourceRequirements,
-		storages []kbappsv1.ClusterComponentVolumeClaimTemplate,
+		storages []kbappsv1.PersistentVolumeClaimTemplate,
 		replicas int32,
 		clusterCompName string,
 		templateName string,
@@ -514,12 +514,12 @@ func (o *ClusterObjects) getPodPhase(pod *corev1.Pod) string {
 	return reason
 }
 
-func (o *ClusterObjects) getStorageInfo(vcts []kbappsv1.ClusterComponentVolumeClaimTemplate, componentName string) []StorageInfo {
+func (o *ClusterObjects) getStorageInfo(vcts []kbappsv1.PersistentVolumeClaimTemplate, componentName string) []StorageInfo {
 	if len(vcts) == 0 {
 		return nil
 	}
 
-	getClassName := func(vcTpl *kbappsv1.ClusterComponentVolumeClaimTemplate) string {
+	getClassName := func(vcTpl *kbappsv1.PersistentVolumeClaimTemplate) string {
 		if vcTpl.Spec.StorageClassName != nil {
 			return *vcTpl.Spec.StorageClassName
 		}

pkg/cluster/helper.go

@@ -81,7 +81,7 @@ func getInstanceInfoByList(dynamic dynamic.Interface, name, componentName, names
 		if ok {
 			info.Role = role
 		}
-		if role == constant.Primary || role == constant.Leader {
+		if role == "primary" || role == "leader" {
 			infos = append([]*InstanceInfo{info}, infos...)
 		} else {
 			infos = append(infos, info)

pkg/cmd/cluster/config_edit.go

@@ -28,10 +28,10 @@ import (
 
 	opsv1alpha1 "github.com/apecloud/kubeblocks/apis/operations/v1alpha1"
 	parametersv1alpha1 "github.com/apecloud/kubeblocks/apis/parameters/v1alpha1"
-	cfgcm "github.com/apecloud/kubeblocks/pkg/configuration/config_manager"
-	"github.com/apecloud/kubeblocks/pkg/configuration/core"
-	"github.com/apecloud/kubeblocks/pkg/configuration/validate"
-	intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil"
+	configctrl "github.com/apecloud/kubeblocks/pkg/parameters"
+	cfgcm "github.com/apecloud/kubeblocks/pkg/parameters/configmanager"
+	"github.com/apecloud/kubeblocks/pkg/parameters/core"
+	"github.com/apecloud/kubeblocks/pkg/parameters/validate"
 	"github.com/spf13/cobra"
 	"golang.org/x/exp/slices"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -111,7 +111,7 @@ func hasSchemaForFile(rctx *ReconfigureContext, configFile string) bool {
 	if rctx.ConfigRender == nil {
 		return false
 	}
-	return intctrlutil.GetComponentConfigDescription(&rctx.ConfigRender.Spec, configFile) != nil
+	return configctrl.GetComponentConfigDescription(&rctx.ConfigRender.Spec, configFile) != nil
 }
 
 func (o *editConfigOptions) runWithConfigConstraints(cfgEditContext *configEditContext, rctx *ReconfigureContext, fn func() error) error {
@@ -146,7 +146,7 @@ func (o *editConfigOptions) runWithConfigConstraints(cfgEditContext *configEditC
 	}
 
 	var config *parametersv1alpha1.ComponentConfigDescription
-	if config = intctrlutil.GetComponentConfigDescription(&rctx.ConfigRender.Spec, o.CfgFile); config == nil {
+	if config = configctrl.GetComponentConfigDescription(&rctx.ConfigRender.Spec, o.CfgFile); config == nil {
 		return fn()
 	}
 	var pd *parametersv1alpha1.ParametersDefinition

pkg/cmd/cluster/config_observer.go

@@ -23,16 +23,16 @@ import (
 	"fmt"
 
 	parametersv1alpha1 "github.com/apecloud/kubeblocks/apis/parameters/v1alpha1"
-	intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil"
+	configctrl "github.com/apecloud/kubeblocks/pkg/parameters"
 	"github.com/spf13/cobra"
 	apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/cli-runtime/pkg/genericiooptions"
 	cmdutil "k8s.io/kubectl/pkg/cmd/util"
 	"k8s.io/kubectl/pkg/util/templates"
 
-	cfgcore "github.com/apecloud/kubeblocks/pkg/configuration/core"
-	"github.com/apecloud/kubeblocks/pkg/configuration/openapi"
-	cfgutil "github.com/apecloud/kubeblocks/pkg/configuration/util"
+	cfgcore "github.com/apecloud/kubeblocks/pkg/parameters/core"
+	"github.com/apecloud/kubeblocks/pkg/parameters/openapi"
+	cfgutil "github.com/apecloud/kubeblocks/pkg/parameters/util"
 
 	"github.com/apecloud/kbcli/pkg/printer"
 	"github.com/apecloud/kbcli/pkg/types"
@@ -140,7 +140,7 @@ func (r *configObserverOptions) printComponentConfigSpecsDescribe(rctx *Reconfig
 func (r *configObserverOptions) printComponentExplainConfigure(rctx *ReconfigureContext) error {
 	for _, pd := range rctx.ParametersDefs {
 		if rctx.ConfigRender != nil {
-			config := intctrlutil.GetComponentConfigDescription(&rctx.ConfigRender.Spec, pd.Spec.FileName)
+			config := configctrl.GetComponentConfigDescription(&rctx.ConfigRender.Spec, pd.Spec.FileName)
 			if config != nil {
 				fmt.Println("template meta:")
 				printer.PrintLineWithTabSeparator(

pkg/cmd/cluster/config_ops.go

@@ -27,11 +27,10 @@ import (
 	opsv1alpha1 "github.com/apecloud/kubeblocks/apis/operations/v1alpha1"
 	parametersv1alpha1 "github.com/apecloud/kubeblocks/apis/parameters/v1alpha1"
 	"github.com/apecloud/kubeblocks/pkg/client/clientset/versioned"
-	cfgcm "github.com/apecloud/kubeblocks/pkg/configuration/config_manager"
-	"github.com/apecloud/kubeblocks/pkg/configuration/core"
-	configctrl "github.com/apecloud/kubeblocks/pkg/controller/configuration"
-	controllerutil "github.com/apecloud/kubeblocks/pkg/controllerutil"
 	"github.com/apecloud/kubeblocks/pkg/generics"
+	configctrl "github.com/apecloud/kubeblocks/pkg/parameters"
+	cfgcm "github.com/apecloud/kubeblocks/pkg/parameters/configmanager"
+	"github.com/apecloud/kubeblocks/pkg/parameters/core"
 	"github.com/spf13/cobra"
 	"k8s.io/cli-runtime/pkg/genericiooptions"
 	cmdutil "k8s.io/kubectl/pkg/cmd/util"
@@ -115,7 +114,7 @@ func (o *configOpsOptions) validateReconfigureOptions() error {
 		if err != nil {
 			return err
 		}
-		o.KeyValues = core.FromStringPointerMap(kvs)
+		o.KeyValues = fromStringPointerMap(kvs)
 	}
 	return nil
 }
@@ -174,7 +173,7 @@ func (o *configOpsOptions) validateConfigParams(rctx *ReconfigureContext, classi
 	transform := func(params map[string]*parametersv1alpha1.ParametersInFile) []core.ParamPairs {
 		var result []core.ParamPairs
 		for file, ps := range params {
-			configDescs := controllerutil.GetComponentConfigDescriptions(&rctx.ConfigRender.Spec, file)
+			configDescs := configctrl.GetComponentConfigDescriptions(&rctx.ConfigRender.Spec, file)
 			builder := configctrl.NewValueManager(rctx.ParametersDefs, configDescs)
 			updatedParams, _ := core.FromStringMap(ps.Parameters, builder.BuildValueTransformer(file))
 			result = append(result, core.ParamPairs{
@@ -187,7 +186,7 @@ func (o *configOpsOptions) validateConfigParams(rctx *ReconfigureContext, classi
 
 	restart := false
 	for _, parameters := range classifyParameters {
-		_, err := controllerutil.MergeAndValidateConfigs(mockEmptyData(parameters), transform(parameters), rctx.ParametersDefs, rctx.ConfigRender.Spec.Configs)
+		_, err := configctrl.MergeAndValidateConfigs(mockEmptyData(parameters), transform(parameters), rctx.ParametersDefs, rctx.ConfigRender.Spec.Configs)
 		if err != nil {
 			return err
 		}
@@ -210,6 +209,15 @@ func mockEmptyData(m map[string]*parametersv1alpha1.ParametersInFile) map[string
 	return r
 }
 
+func fromStringPointerMap(m map[string]string) map[string]*string {
+	r := make(map[string]*string, len(m))
+	for key, value := range m {
+		v := value
+		r[key] = &v
+	}
+	return r
+}
+
 func (o *configOpsOptions) confirmReconfigureWithRestart() error {
 	if o.AutoApprove {
 		return nil

pkg/cmd/cluster/config_ops_test.go

@@ -34,7 +34,7 @@ import (
 
 	kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1"
 	kbfakeclient "github.com/apecloud/kubeblocks/pkg/client/clientset/versioned/fake"
-	cfgcore "github.com/apecloud/kubeblocks/pkg/configuration/core"
+	cfgcore "github.com/apecloud/kubeblocks/pkg/parameters/core"
 	testapps "github.com/apecloud/kubeblocks/pkg/testutil/apps"
 
 	"github.com/apecloud/kbcli/pkg/testing"
@@ -79,7 +79,7 @@ var _ = Describe("reconfigure test", func() {
 
 		By("Create configmap and config constraint obj")
 		configmap := testapps.NewCustomizedObj("resources/mysql-config-template.yaml", &corev1.ConfigMap{}, testapps.WithNamespace(ns), testapps.WithName(testing.FakeMysqlTemplateName))
-		componentConfig := testapps.NewConfigMap(ns, cfgcore.GetComponentCfgName(clusterName, statefulCompName, configSpecName), testapps.SetConfigMapData("my.cnf", ""))
+		componentConfig := testapps.NewConfigMap(ns, cfgcore.GetComponentCfgName(clusterName, statefulCompName, configSpecName), setConfigMapData("my.cnf", ""))
 		objs := []runtime.Object{configmap, componentConfig}
 		ttf, ops := NewFakeOperationsOptions(ns, clusterName, objs...)
 		o := &configOpsOptions{
@@ -121,3 +121,9 @@ var _ = Describe("reconfigure test", func() {
 	})
 
 })
+
+func setConfigMapData(key string, value string) func(*corev1.ConfigMap) {
+	return func(cm *corev1.ConfigMap) {
+		cm.Data[key] = value
+	}
+}

pkg/cmd/cluster/config_util.go

@@ -38,8 +38,8 @@ import (
 	"k8s.io/kubectl/pkg/cmd/util/editor"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	cfgcore "github.com/apecloud/kubeblocks/pkg/configuration/core"
-	cfgutil "github.com/apecloud/kubeblocks/pkg/configuration/util"
+	cfgcore "github.com/apecloud/kubeblocks/pkg/parameters/core"
+	cfgutil "github.com/apecloud/kubeblocks/pkg/parameters/util"
 
 	"github.com/apecloud/kbcli/pkg/action"
 	"github.com/apecloud/kbcli/pkg/printer"

pkg/cmd/cluster/config_wrapper.go

@@ -27,8 +27,8 @@ import (
 	kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1"
 	parametersv1alpha1 "github.com/apecloud/kubeblocks/apis/parameters/v1alpha1"
 	"github.com/apecloud/kubeblocks/pkg/client/clientset/versioned"
-	intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil"
 	"github.com/apecloud/kubeblocks/pkg/generics"
+	configctrl "github.com/apecloud/kubeblocks/pkg/parameters"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	cmdutil "k8s.io/kubectl/pkg/cmd/util"
 )
@@ -61,7 +61,7 @@ func (w *ReconfigureWrapper) ConfigSpecName() string {
 	}
 	file := w.ConfigFile()
 	if file != "" && w.rctx.ConfigRender != nil {
-		config := intctrlutil.GetComponentConfigDescription(&w.rctx.ConfigRender.Spec, file)
+		config := configctrl.GetComponentConfigDescription(&w.rctx.ConfigRender.Spec, file)
 		if config != nil {
 			return config.TemplateName
 		}

pkg/cmd/cluster/errors.go

@@ -20,7 +20,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package cluster
 
 import (
-	cfgcore "github.com/apecloud/kubeblocks/pkg/configuration/core"
+	cfgcore "github.com/apecloud/kubeblocks/pkg/parameters/core"
 )
 
 var (

pkg/cmd/cluster/operations.go

@@ -266,7 +266,7 @@ func (o *OperationsOptions) validateVolumeExpansion(clusterObj *appsv1.Cluster)
 			}
 			var pvc *corev1.PersistentVolumeClaim
 			for _, pvcItem := range pvcs.Items {
-				if pvcItem.Labels[constant.KBAppComponentInstanceTemplateLabelKey] == "" {
+				if pvcItem.Labels["apps.kubeblocks.io/instance-template"] == "" {
 					pvc = &pvcItem
 					break
 				}

pkg/cmd/cluster/update.go

@@ -39,9 +39,9 @@ import (
 	"k8s.io/kubectl/pkg/util/templates"
 
 	dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1"
-	cfgutil "github.com/apecloud/kubeblocks/pkg/configuration/util"
 	"github.com/apecloud/kubeblocks/pkg/constant"
 	"github.com/apecloud/kubeblocks/pkg/dataprotection/utils"
+	cfgutil "github.com/apecloud/kubeblocks/pkg/parameters/util"
 
 	"github.com/apecloud/kbcli/pkg/action"
 	"github.com/apecloud/kbcli/pkg/cluster"

pkg/cmd/kubeblocks/kubeblocks_objects_test.go

@@ -232,6 +232,6 @@ func mockConfigMaps() []runtime.Object {
 	if obj.ObjectMeta.Labels == nil {
 		obj.ObjectMeta.Labels = make(map[string]string)
 	}
-	obj.ObjectMeta.Labels[constant.CMConfigurationTypeLabelKey] = constant.ConfigTemplateType
+	obj.ObjectMeta.Labels[constant.CMConfigurationTypeLabelKey] = "tpl"
 	return []runtime.Object{obj}
 }

pkg/cmd/kubeblocks/util.go

@@ -201,7 +201,7 @@ func buildKubeBlocksSelectorLabels() string {
 // and it should be consistent with the labelSelectors defined
 // in `configuration.updateConfigMapFinalizerImpl`.
 func buildConfigTypeSelectorLabels() string {
-	return fmt.Sprintf("%s=%s", constant.CMConfigurationTypeLabelKey, constant.ConfigTemplateType)
+	return fmt.Sprintf("%s=%s", constant.CMConfigurationTypeLabelKey, "tpl")
 }
 
 // printAddonMsg prints addon message when has failed addon or timeouts

pkg/testing/fake.go

@@ -124,10 +124,10 @@ func FakeCluster(name, namespace string, conditions ...metav1.Condition) *kbapps
 							corev1.ResourceMemory: resource.MustParse("2Gi"),
 						},
 					},
-					VolumeClaimTemplates: []kbappsv1.ClusterComponentVolumeClaimTemplate{
+					VolumeClaimTemplates: []kbappsv1.PersistentVolumeClaimTemplate{
 						{
 							Name: "data",
-							Spec: kbappsv1.PersistentVolumeClaimSpec{
+							Spec: corev1.PersistentVolumeClaimSpec{
 								AccessModes: []corev1.PersistentVolumeAccessMode{
 									corev1.ReadWriteOnce,
 								},
@@ -150,10 +150,10 @@ func FakeCluster(name, namespace string, conditions ...metav1.Condition) *kbapps
 							corev1.ResourceMemory: resource.MustParse("100Mi"),
 						},
 					},
-					VolumeClaimTemplates: []kbappsv1.ClusterComponentVolumeClaimTemplate{
+					VolumeClaimTemplates: []kbappsv1.PersistentVolumeClaimTemplate{
 						{
 							Name: "data",
-							Spec: kbappsv1.PersistentVolumeClaimSpec{
+							Spec: corev1.PersistentVolumeClaimSpec{
 								AccessModes: []corev1.PersistentVolumeAccessMode{
 									corev1.ReadWriteOnce,
 								},

pkg/util/conversion/conversion.go

@@ -21,7 +21,6 @@ import (
 
 	appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1"
 	appsv1beta1 "github.com/apecloud/kubeblocks/apis/apps/v1beta1"
-	"github.com/apecloud/kubeblocks/pkg/constant"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -37,6 +36,10 @@ import (
 const (
 	OldVersion = "08"
 	NewVersion = "09"
+
+	kubeblocksAPIConversionTypeAnnotationName = "api.kubeblocks.io/converted"
+	sourceAPIVersionAnnotationName            = "api.kubeblocks.io/source"
+	migratedAPIVersion                        = "migrated"
 )
 
 func FetchAndConversionResources(versionMeta *VersionConversionMeta) ([]unstructured.Unstructured, error) {
@@ -103,7 +106,7 @@ func hasConversionVersion(obj client.Object) bool {
 	if len(annotations) == 0 {
 		return false
 	}
-	return annotations[constant.KubeblocksAPIConversionTypeAnnotationName] == constant.MigratedAPIVersion
+	return annotations[kubeblocksAPIConversionTypeAnnotationName] == migratedAPIVersion
 }
 
 func UpdateNewVersionResources(versionMeta *VersionConversionMeta, targetObjects []unstructured.Unstructured) error {
@@ -152,8 +155,8 @@ func convertImpl(source *appsv1alpha1.ConfigConstraint, target *appsv1beta1.Conf
 	if target.Annotations == nil {
 		target.Annotations = make(map[string]string)
 	}
-	target.Annotations[constant.KubeblocksAPIConversionTypeAnnotationName] = constant.MigratedAPIVersion
-	target.Annotations[constant.SourceAPIVersionAnnotationName] = appsv1alpha1.GroupVersion.Version
+	target.Annotations[kubeblocksAPIConversionTypeAnnotationName] = migratedAPIVersion
+	target.Annotations[sourceAPIVersionAnnotationName] = appsv1alpha1.GroupVersion.Version
 	convertToConstraintSpec(&source.Spec, &target.Spec)
 	return nil
 }

pkg/util/util.go

@@ -77,10 +77,10 @@ import (
 
 	kbappsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1"
 	kbappsv1beta1 "github.com/apecloud/kubeblocks/apis/apps/v1beta1"
-	"github.com/apecloud/kubeblocks/pkg/configuration/core"
-	"github.com/apecloud/kubeblocks/pkg/configuration/openapi"
-	cfgutil "github.com/apecloud/kubeblocks/pkg/configuration/util"
 	"github.com/apecloud/kubeblocks/pkg/constant"
+	"github.com/apecloud/kubeblocks/pkg/parameters/core"
+	"github.com/apecloud/kubeblocks/pkg/parameters/openapi"
+	cfgutil "github.com/apecloud/kubeblocks/pkg/parameters/util"
 	viper "github.com/apecloud/kubeblocks/pkg/viperx"
 
 	"github.com/apecloud/kbcli/pkg/testing"

pkg/util/util_test.go

@@ -41,8 +41,8 @@ import (
 
 	appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1"
 	appsv1beta1 "github.com/apecloud/kubeblocks/apis/apps/v1beta1"
-	cfgcore "github.com/apecloud/kubeblocks/pkg/configuration/core"
 	"github.com/apecloud/kubeblocks/pkg/constant"
+	cfgcore "github.com/apecloud/kubeblocks/pkg/parameters/core"
 	testapps "github.com/apecloud/kubeblocks/pkg/testutil/apps"
 	"github.com/apecloud/kubeblocks/test/testdata"
 
@@ -272,7 +272,7 @@ var _ = Describe("util", func() {
 		}}
 
 		for _, tt := range tests {
-			ret, err := IsSupportReconfigureParams(tt.args.configSpec, cfgcore.FromStringPointerMap(tt.args.updatedParams), mockClient)
+			ret, err := IsSupportReconfigureParams(tt.args.configSpec, fromStringPointerMap(tt.args.updatedParams), mockClient)
 			Expect(ret).Should(BeEquivalentTo(tt.expected))
 			Expect(err != nil).Should(BeEquivalentTo(tt.err != nil))
 			if err != nil {
@@ -364,3 +364,12 @@ var _ = Describe("util", func() {
 		}
 	})
 })
+
+func fromStringPointerMap(m map[string]string) map[string]*string {
+	r := make(map[string]*string, len(m))
+	for key, value := range m {
+		v := value
+		r[key] = &v
+	}
+	return r
+}