1. support more parameters of neonsan create_volume

2. judge if device readable, before mount
3. keep idempotent of node operation
4. add correct parameters for clone
5. fix documents error
6. add install script of neonsan-plugin

Makefile

@@ -44,7 +44,7 @@ mod:
 	go mod vendor
 
 test:
-	go test -cover -mod=vendor -gcflags=-l ./pkg/...
+	go test -cover -mod=vendor -gcflags=-l -count=1 ./pkg/...
 
 clean:
 	go clean -r -x ./...

README.md

@@ -135,14 +135,14 @@ StorageClass definition [file](deploy/neonsan/example/volume/sc.yaml) shown belo
   provisioner: neonsan.csi.qingstor.com
   parameters:
     fsType: "ext4"
-    replica: "2"
-    pool: "kube"
+    rep_count: "2"
+    pool_name: "kube"
   reclaimPolicy: Delete 
 ```
 
 - `fsType`: `ext3`, `ext4`, `xfs`. Default `ext4`.
-- `replica`: count of replicas (`1-3`). Default` 1`.
-- `poolName`: pool of Neonsan, should not be empty. 
+- `rep_count`: count of replicas (`1-3`). Default` 1`.
+- `pool_name`: pool name of Neonsan, should not be empty. 
 
 ## Document
 - [User Guide](docs/user-guide.md)

deploy/neonsan/example/volume/sc.yaml

@@ -22,6 +22,6 @@ provisioner: neonsan.csi.qingstor.com
 parameters:
   fsType: "ext4"
   replica: "1"
-  pool: "kube"
+  pool_name: "kube"
 reclaimPolicy: Delete
 allowVolumeExpansion: true

deploy/neonsan/plugin/install-by-kubectl-cp.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+status=$(systemctl is-active neonsan-plugin)
+if [[ ${status} = "active" ]]; then
+  echo "neonsan-plugin is active, stop it "
+  systemctl stop neonsan-plugin
+fi
+
+podName=$(kubectl -n kube-system get pod -l app=csi-neonsan,role=controller --field-selector=status.phase=Running -o name |  head -1 | awk -F'/' '{print $2}')
+if test -z ${podName}; then
+  echo "no controller pod, failed to start, please check"
+  exit 1
+fi
+
+kubectl -n kube-system -c csi-neonsan cp ${podName}:neonsan-csi-driver /usr/bin/neonsan-plugin
+chmod +x /usr/bin/neonsan-plugin
+
+cat > /etc/systemd/system/neonsan-plugin.service  <<EOF
+[Unit]
+Description=NeonSAN CSI Plugin
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/neonsan-plugin  --endpoint=unix:///var/lib/kubelet/plugins/neonsan.csi.qingstor.com/csi.sock
+
+Restart=on-failure
+StartLimitBurst=3
+StartLimitInterval=60s
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl daemon-reload
+systemctl enable neonsan-plugin
+systemctl start neonsan-plugin
+
+status=$(systemctl is-active neonsan-plugin)
+if [[ ${status} = "active" ]]; then
+  echo "neonsan-plugin start successfully"
+else
+  echo "neonsan-plugin failed to start, please check"
+fi

docs/user-guide.md

@@ -11,8 +11,8 @@ metadata:
 provisioner: neonsan.csi.qingstor.com
 parameters:
   fsType: "ext4"
-  replica: "1"
-  pool: "kube"
+  rep_count: "1"
+  pool_name: "kube"
 reclaimPolicy: Delete
 allowVolumeExpansion: true
 ```
@@ -22,11 +22,11 @@ allowVolumeExpansion: true
 #### `fsType`
 Support `ext3`, `ext4`, `xfs`, default `ext4`.
 
-#### `replica`
+#### `rep_count`
 Number of disk replicas, default `1`，maximum `3`.
 
-#### `pool`
-Neonsan pool，not empty
+#### `pool_name`
+Neonsan pool name，not empty
 
 ### Other Parameters 
 
@@ -107,7 +107,7 @@ This plugin only supports offline volume expansion. The procedure of offline vol
 1. Ensure volume in unmounted status
 2. Edit the capacity of PVC
 3. Mount volume on workload
-Please reference [Example YAML files](https://github.com/yunify/qingcloud-csi/tree/master/deploy/disk/volume).
+Please reference [Example YAML files](deploy/neonsan/example/volume).
 
 ### Prerequisite
 - Kubernetes 1.14+ cluster
@@ -183,7 +183,7 @@ pvc-clone   Bound    pvc-a75e3f7c-59af-43ef-82d3-300508871432   20Gi       RWO
 ```
 
 ## Snapshot Management
-Snapshot management contains creating/deleting snapshot and restoring volume from snapshpot. Please reference [Example YAML files](https://github.com/yunify/qingcloud-csi/tree/master/deploy/disk/example/snapshot).
+Snapshot management contains creating/deleting snapshot and restoring volume from snapshot. Please reference [Example YAML files](deploy/neonsan/example/snapshot).
 
 ### Prerequisites
 - Kubernetes 1.14+ 

helm/csi-neonsan/values.yaml

@@ -36,6 +36,3 @@ snapshotter:
 registrar:
   repository: csiplugin/csi-node-driver-registrar
   tag: v1.2.0
-snapshotController:
-  repository: csiplugin/snapshot-controller
-  tag: v2.0.1

pkg/service/controller.go

@@ -92,7 +92,7 @@ func (s *service) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest
 		}
 	}
 	if createError != nil {
-		klog.Errorf("Failed to create volume %s, contentSource %s, error: %v", req.GetVolumeContentSource(), volumeName, err)
+		klog.Errorf("Failed to create volume %s, contentSource %s, error: %v", volumeName, req.GetVolumeContentSource(), createError)
 		return nil, status.Error(codes.Internal, createError.Error())
 	}
 	csiVolume := &csi.Volume{
@@ -220,7 +220,7 @@ func (s *service) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotReq
 	if snapshot != nil {
 		return &csi.CreateSnapshotResponse{Snapshot: snapshot}, nil
 	}
-	return nil, status.Errorf(codes.Internal, "not find after create snapshot : %s", snapName, )
+	return nil, status.Errorf(codes.Internal, "not find after create snapshot : %s", snapName)
 }
 
 // CreateSnapshot allows the CO to delete a snapshot.

pkg/service/node.go

@@ -52,15 +52,24 @@ func (s *service) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeR
 	if !notMnt {
 		return &csi.NodeStageVolumeResponse{}, nil
 	}
-	// Attach if need
-	err = s.storageProvider.NodeAttachVolume(volumeID)
-	if err != nil {
-		return nil, status.Error(codes.Internal, err.Error())
-	}
+
+	// for idempotent, if device not empty, volume has already attached
 	devicePath, err := s.storageProvider.NodeGetDevice(volumeID)
 	if err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
 	}
+	if len(devicePath) == 0 {
+		// Attach if need
+		err = s.storageProvider.NodeAttachVolume(volumeID)
+		if err != nil {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+		devicePath, err = s.storageProvider.NodeGetDevice(volumeID)
+		if err != nil {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+	}
+
 	// do mount
 	if err := s.mounter.FormatAndMount(devicePath, targetPath, fsType, []string{}); err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
@@ -109,6 +118,16 @@ func (s *service) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVol
 		klog.Errorf("Volume %s still mounted in instance %s", volumeID, s.option.NodeId)
 		return nil, status.Error(codes.Internal, "unmount failed")
 	}
+
+	devicePath, err := s.storageProvider.NodeGetDevice(volumeID)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	// for idempotent, if device is empty, the volume has already detached
+	if len(devicePath) == 0 {
+		return &csi.NodeUnstageVolumeResponse{}, nil
+	}
+
 	// node detach volume
 	err = s.storageProvider.NodeDetachVolume(volumeID)
 	if err != nil {

pkg/storage/neonsan/api/api.go

@@ -44,12 +44,28 @@ type Volume struct {
 	ReplicationCount    int       `json:"replication_count"`
 	Status              string    `json:"status"`
 	MinReplicationCount int       `json:"min_replication_count"`
+	Role                string    `json:"role"`
+	Policy              string    `json:"policy"`
 	CreateTime          time.Time `json:"create_time" format:"ISO 8601"`
 	StatusTime          time.Time `json:"status_time" format:"ISO 8601"`
 	MetroReplica        string    `json:"metro_replica"`
+	ProvisionType       string    `json:"provision_type"` // thin or thick
+	MaxBs               int       `json:"max_bs"`
 	VolumeAllocated     int       `json:"volume_allocated"`
-	ProvisionType       string    `json:"provision_type"`
-	Role                string    `json:"role"`
+	RgName              string    `json:"rg_name"`
+	Encrypted           string    `json:"encrypted"`
+}
+
+type VolumeForClone struct {
+	ID                  int    `json:"id"`
+	Size                int    `json:"size"`
+	ReplicationCount    int    `json:"replication_count"`
+	MinReplicationCount int    `json:"min_replication_count"`
+	Role                string `json:"role"`
+	MaxBs               int    `json:"max_bs"`
+	Encrypte            string `json:"encrypte"`
+	KeyName             string `json:"key_name"`
+	Rg                  string `json:"rg"`
 }
 
 func ListVolume(confFile, poolName, volumeName string) (*Volume, error) {
@@ -72,13 +88,12 @@ func ListVolume(confFile, poolName, volumeName string) (*Volume, error) {
 	return &response.Volumes[0], nil
 }
 
-func CreateVolume(confFile, poolName, volumeName string, size int64, repCount int) error {
+func CreateVolume(confFile, volumeName string, size int64, parameters map[string]string) error {
 	request := CreateVolumeRequest{
-		Op:       "create_volume",
-		PoolName: poolName,
-		Name:     volumeName,
-		Size:     size,
-		RepCount: repCount,
+		Op:         "create_volume",
+		Name:       volumeName,
+		Size:       size,
+		Parameters: parameters,
 	}
 	response := &CreateVolumeResponse{}
 	return httpGet(confFile, request, response)
@@ -105,6 +120,20 @@ func ResizeVolume(confFile, poolName, volumeName string, size int64) error {
 	return httpGet(confFile, request, response)
 }
 
+func GetVolumeForClone(confFile, poolName, volumeName string) (*VolumeForClone, error) {
+	request := GetVolumeForCloneRequest{
+		Op:       "get_volume_info",
+		PoolName: poolName,
+		Name:     volumeName,
+	}
+	response := &GetVolumeForCloneResponse{}
+	err := httpGet(confFile, request, response)
+	if err != nil {
+		return nil, err
+	}
+	return &response.VolumeInfo, nil
+}
+
 func CloneVolume(confFile, sourcePoolName, sourceVolumeName, snapshotName, targetVolumeName, targetPoolName string) error {
 	request := CloneVolumeRequest{
 		Op:           "clone_volume",
@@ -136,9 +165,9 @@ func ListClone(confFile, sourcePoolName, sourceVolumeName, targetPoolName, targe
 
 func ListClone220(confFile, sourcePoolName, sourceVolumeName, targetPoolName, targetVolumeName string) (*CloneInfo, error) {
 	request := ListCloneRequest220{
-		Op:           "list_clone",
+		Op:        "list_clone",
 		SourceVol: sourcePoolName + "/" + sourceVolumeName,
-		TargetVol: targetPoolName+ "/" + targetVolumeName,
+		TargetVol: targetPoolName + "/" + targetVolumeName,
 	}
 	response := &ListCloneResponse{}
 	err := httpGet(confFile, request, response)
@@ -151,7 +180,6 @@ func ListClone220(confFile, sourcePoolName, sourceVolumeName, targetPoolName, ta
 	return &response.CloneVolumes[0], nil
 }
 
-
 func DetachCloneRelationship(confFile, sourcePoolName, sourceVolumeName, targetPoolName, targetVolumeName string) error {
 	request := DetachCloneRelationshipRequest{
 		Op:        "detach_clone_relationship",
@@ -208,20 +236,25 @@ func ListSnapshot(confFile, poolName, volumeName, snapshotName string) (*Snapsho
 func buildParameters(request interface{}) string {
 	t, v := reflect.TypeOf(request), reflect.ValueOf(request)
 	sb := strings.Builder{}
+	parameter := make(map[string]string)
 	for k := 0; k < t.NumField(); k++ {
-		sb.WriteString(t.Field(k).Tag.Get(`json`))
-		sb.WriteString("=")
 		switch val := v.Field(k).Interface().(type) {
 		case int:
-			sb.WriteString(strconv.Itoa(val))
+			parameter[t.Field(k).Tag.Get("json")] = strconv.Itoa(val)
 		case int64:
-			sb.WriteString(strconv.Itoa(int(val)))
+			parameter[t.Field(k).Tag.Get("json")] = strconv.Itoa(int(val))
 		case string:
-			sb.WriteString(val)
+			parameter[t.Field(k).Tag.Get("json")] = val
+		case map[string]string:
+			for k1, v1 := range val {
+				parameter[k1] = v1
+			}
 		default:
 			klog.Warning("invalidType: ", reflect.TypeOf(val))
 		}
-		sb.WriteString("&")
+	}
+	for k2, v2 := range parameter {
+		sb.WriteString(fmt.Sprintf("%s=%s&", k2, v2))
 	}
 	return sb.String()
 }
@@ -238,18 +271,20 @@ func httpGet(confFile string, request interface{}, response Response) error {
 	if err != nil {
 		return err
 	}
-	if ret.StatusCode != 200 {
-		return fmt.Errorf("NeonsanAPI http code:%d", ret.StatusCode)
-	}
 	defer func() {
-		_ = ret.Body.Close()
+		if ret != nil && ret.Body != nil {
+			_ = ret.Body.Close()
+		}
 	}()
-
+	// http 400 is param error, let it show ret.Body in error
+	if ret.StatusCode != 200 && ret.StatusCode != 400 {
+		return fmt.Errorf("NeonsanAPI http code:%d", ret.StatusCode)
+	}
 	body, err := ioutil.ReadAll(ret.Body)
 	if err != nil {
 		return err
 	}
-	klog.Infof("NeonsanApi response:%s, request:%s",  body, url)
+	klog.Infof("NeonsanApi response:%s, request:%s", body, url)
 	err = json.Unmarshal(body, response)
 	if err != nil {
 		return err

pkg/storage/neonsan/api/api_test.go

@@ -38,9 +38,8 @@ var (
 
 func TestHttpGet(t *testing.T) {
 	request := CreateVolumeRequest{
-		Op:       "list_volume",
-		PoolName: "",
-		Name:     "",
+		Op:   "list_volume",
+		Name: "",
 	}
 	response := &CreateVolumeResponse{}
 	var err error
@@ -194,14 +193,14 @@ func TestCreateVolume(t *testing.T) {
 	})
 	defer guardHttpGetOK.Unpatch()
 
-	err := CreateVolume(configFile, poolName, "xx", 1, 1)
+	err := CreateVolume(configFile, "xx", 1, nil)
 	convey.Convey("create volume success", t, func() {
 		convey.So(err, convey.ShouldBeNil)
 	})
 
 	mockResponse.RetCode = 100
 	httpBody, _ = json.Marshal(mockResponse)
-	err = CreateVolume(configFile, poolName, "xx", 1, 1)
+	err = CreateVolume(configFile, "xx", 1, nil)
 	convey.Convey("create volume fail", t, func() {
 		convey.So(err, convey.ShouldNotBeNil)
 	})
@@ -243,4 +242,3 @@ func TestDeleteVolume(t *testing.T) {
 	})
 
 }
-