Index: main.py
===================================================================
--- main.py	(revision 6cfa913dc64bda1a424d6822e0fc38cc2775eb77)
+++ main.py	(revision 7c6bfbb0aa0df9d1b5db80eef7cd52e38092e80b)
@@ -9,21 +9,15 @@
 import random
 import numpy as np
-from scipy import exp
-from scipy.optimize import leastsq
-from scipy.optimize import curve_fit
-import matplotlib.pyplot as plt
-import matplotlib.animation as anim
 from drawnow import drawnow
 import warnings
-import signal
 import json
 from Queue import Queue
-from threading import Thread
 import threading
 import logging
 import multiprocessing
-import paramiko
 import fcntl
 import logging
+import subprocess
+import shlex
 
 #write = sys.stdout.write
@@ -39,4 +33,98 @@
 logger.addHandler(hdlr)
 logger.setLevel(logging.DEBUG)
+
+GPPDgroup1=[]
+GPPDgroup1.append('module521')
+GPPDgroup1.append('module522')
+GPPDgroup1.append('module523')
+GPPDgroup1.append('module531')
+GPPDgroup1.append('module532')
+GPPDgroup1.append('module533')
+GPPDgroup1.append('module541')
+GPPDgroup1.append('module542')
+GPPDgroup1.append('module543')
+
+GPPDgroup2=[]
+GPPDgroup2.append('module621')
+GPPDgroup2.append('module622')
+GPPDgroup2.append('module623')
+GPPDgroup2.append('module631')
+GPPDgroup2.append('module632')
+GPPDgroup2.append('module633')
+GPPDgroup2.append('module641')
+GPPDgroup2.append('module642')
+GPPDgroup2.append('module643')
+
+GPPDgroup3=[]
+GPPDgroup3.append('module131')
+GPPDgroup3.append('module132')
+GPPDgroup3.append('module133')
+GPPDgroup3.append('module231')
+GPPDgroup3.append('module232')
+GPPDgroup3.append('module233')
+GPPDgroup3.append('module322')
+GPPDgroup3.append('module331')
+GPPDgroup3.append('module332')
+GPPDgroup3.append('module333')
+GPPDgroup3.append('module341')
+GPPDgroup3.append('module342')
+GPPDgroup3.append('module343')
+GPPDgroup3.append('module422')
+GPPDgroup3.append('module431')
+GPPDgroup3.append('module432')
+GPPDgroup3.append('module433')
+GPPDgroup3.append('module441')
+GPPDgroup3.append('module442')
+GPPDgroup3.append('module443')
+
+GPPDgroup4=[]
+GPPDgroup4.append('monitor1')
+GPPDgroup4.append('monitor2')
+GPPDgroup4.append('monitor3')
+
+GPPDdroneInfo=[]
+_n=-1
+for i in range(len(GPPDgroup1)):
+    _n+=1
+    GPPDdroneInfo.append({})
+    GPPDdroneInfo[_n]['host']='10.1.33.143'
+    GPPDdroneInfo[_n]['module']=GPPDgroup1[i]
+    GPPDdroneInfo[_n]['pid']=-1
+    GPPDdroneInfo[_n]['running']='0'
+    GPPDdroneInfo[_n]['rpc']='0'
+    GPPDdroneInfo[_n]['pobox']='0'
+    GPPDdroneInfo[_n]['heartbeat']='/GPPD/heartbeat/detector/'+GPPDdroneInfo[_n]['host']+':'+GPPDgroup1[i]
+for i in range(len(GPPDgroup2)):
+    _n+=1
+    GPPDdroneInfo.append({})
+    GPPDdroneInfo[_n]['host']='10.1.33.143'
+    GPPDdroneInfo[_n]['module']=GPPDgroup2[i]
+    GPPDdroneInfo[_n]['pid']=-1
+    GPPDdroneInfo[_n]['running']='0'
+    GPPDdroneInfo[_n]['rpc']='0'
+    GPPDdroneInfo[_n]['pobox']='0'
+    GPPDdroneInfo[_n]['heartbeat']='/GPPD/heartbeat/detector/'+GPPDdroneInfo[_n]['host']+':'+GPPDgroup2[i]
+
+for i in range(len(GPPDgroup3)):
+    _n+=1
+    GPPDdroneInfo.append({})
+    GPPDdroneInfo[_n]['host']='10.1.33.143'
+    GPPDdroneInfo[_n]['module']=GPPDgroup3[i]
+    GPPDdroneInfo[_n]['pid']=-1
+    GPPDdroneInfo[_n]['running']='0'
+    GPPDdroneInfo[_n]['rpc']='0'
+    GPPDdroneInfo[_n]['pobox']='0'
+    GPPDdroneInfo[_n]['heartbeat']='/GPPD/heartbeat/detector/'+GPPDdroneInfo[_n]['host']+':'+GPPDgroup3[i]
+
+for i in range(len(GPPDgroup4)):
+    _n+=1
+    GPPDdroneInfo.append({})
+    GPPDdroneInfo[_n]['host']='10.1.33.144'
+    GPPDdroneInfo[_n]['module']=GPPDgroup4[i]
+    GPPDdroneInfo[_n]['pid']=-1
+    GPPDdroneInfo[_n]['running']='0'
+    GPPDdroneInfo[_n]['rpc']='0'
+    GPPDdroneInfo[_n]['pobox']='0'
+    GPPDdroneInfo[_n]['heartbeat']='/GPPD/heartbeat/detector/'+GPPDdroneInfo[_n]['host']+':'+GPPDgroup4[i]
 
 fh=0
@@ -96,40 +184,4 @@
         return self.redisRead
 
-class getNeonServer():
-    def __init__(
-        self,
-        ip,
-        port,
-        timeout,
-        ):
-
-        self.status=False
-        client=False
-        self.server=None
-        begin=time.time()
-        while True:
-            if not self.server:
-                self.server = neon.Neon.NeonRedis(host=ip, port=port, password="sanlie;123", db = 0, isWritable = True)
-            else:
-                try:
-                    client=self.server.client_list()
-                except (redis.exceptions.ConnectionError):
-                    logger.warning("Attempt to connect NEON again!")
-                if client:
-                    self.status = True
-                    logger.info("NEON connected")
-                    break
-                else:
-                    logger.error("NEON not available")
-                if time.time()-begin>timeout:
-                    logger.warning("NEON connect timeout")
-                    break
-
-    def getStatus(self):
-        return self.status
-
-    def getServer(self):
-        return self.server
-
 class getEpicsServer():
     def __init__(
@@ -165,75 +217,117 @@
         return self.epicsCommand, self.epicsStatus
 
-class getSshServer():
-    def __init__(self, ip, user, retry=5):
-        self.server = None
-        self.status=False
-
-        for i in range(retry):
-            if not self.server:
-                try:
-                    _client = paramiko.SSHClient()
-                    _client.load_system_host_keys()    
-                    #ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-                    _client.connect(ip, username=user)
-                    #_client = _client.get_transport()
-                    #_client = _client.open_session()
-                    self.status=True
-                    self.server=_client
-                    break
-                except paramiko.AuthenticationException:
-                    logger.warning("Authentication failed, %s" % ip)
-                except:
-                    logger.warning("Attempt to connect NEON again, %s" % ip)
-                time.sleep(1.0)
-
-    def getStatus(self):
-        return self.status
-
-    def getServer(self):
-        return self.server
-
 class getDroneServer():
-    def __init__(self, ip, module):
-        self.module = module
-        _ssh=getSshServer('10.1.33.143', 'drone', 1)
-        _sshServer = _ssh.getServer()
-        myssh.append(_ssh1Server)
-        myssh.append(_ssh2Server)
-        myssh.append(_ssh3Server)
-        logger.info("ssh Connected")
-
-        # ==================================================
-        #start drone
-        _cmd='nohup /home/drone/Workspace/drone/daqrun/onlinedim.py '+group1[0]+' >'+'/home/drone/Workspace/drone/daqrun/log &'
-        _cmd='/home/drone/Workspace/drone/daqrun/module131'
-        _cmd='/home/drone/Workspace/drone/daqrun/1'
-        stdin, stdout, stderr = myssh[0].exec_command(_cmd)
-        #print myssh[0].get_transport().open_session().exec_command('')
-        #transport = myssh[0].get_transport()
-        #transport.set_keepalive(5) 
-        stdin, stdout, stderr = myssh[0].exec_command('')
-
-        #stdin, stdout, stderr = transport.open_session().exec_command(_cmd)
-        logger.debug(stdout.readlines())
-
+    def __init__(self, droneInfo):
+        import neon
+
+        self.droneInfo=droneInfo
+        self.nModule=len(self.droneInfo)
+
+        self.m_neonRedis = neon.Neon.NeonRedisSentinel([('10.1.33.141', 9001),('10.1.33.145', 9001)], i_socket_timeout=0.1, \
+                                                           master_name='neonmaster', master_timeout = 200, master_password='sanlie;123', \
+                                                           idb = 0, isWritable = True)
+    def create(self, n):
+        while True:
+            _cmd="ssh drone@"+self.droneInfo[n]['host']+" nohup /home/drone/workspace/drone/daqrun/cockpit/"+self.droneInfo[n]['module']
+            _cmd = shlex.split(_cmd)
+            subprocess.Popen(_cmd, shell=False)
+            time.sleep(1.0)
+            _pid=self.getPid(n)
+            if _pid !=-1:
+                clitpob= neon.Neon.NeonService.POBox(self.m_neonRedis, '/GPPD/process/detector', self.droneInfo[n]['host']+':client:'+self.droneInfo[n]['module'])
+                clitrpc = neon.Neon.NeonService.NeonRPC(sendPOBox = clitpob, recvPOBox = clitpob)
+                self.droneInfo[n]['pobox']=clitrpc
+                self.droneInfo[n]['rpc']=self.droneInfo[n]['host']+':server:'+self.droneInfo[n]['module']
+                self.droneInfo[n]['running']='1'
+                self.droneInfo[n]['pid']=_pid
+                break
+
+    def createAll(self):
+        for i in range(self.nModule):
+            self.create(i)
+
+    def kill(self, n):
+        while True:
+            _pid=self.getPid(n)            
+            if _pid!=-1:
+                _cmd="ssh drone@"+self.droneInfo[n]['host']+" /usr/bin/kill -9 "+str(_pid)
+                _cmd = shlex.split(_cmd)
+                subprocess.check_output(_cmd, shell=False)
+                #logger.info("The module killed, %s" % self.droneInfo[n]['module'])
+            else:
+                #logger.warning("The module no exist, %s" % self.droneInfo[n]['module'])
+                break
+
+    def killAll(self):
+        for i in range(self.nModule):
+            self.kill(i)
+
+    def isExist(self, _droneInfo):
+        exist=False
+        _cmd="ssh drone@"+_droneInfo['host']+" ps -ef | grep "+_droneInfo['module']+".py | awk '{print $2}' | head -1"
+        pid=subprocess.check_output(_cmd, shell=True)
+        if pid!='':
+            _cmd="ssh drone@"+_droneInfo['host']+" /usr/bin/kill -9 "+pid
+            _cmd = shlex.split(_cmd)
+            exist=True
+            logger.info("The module exist, %s : %s" % (_droneInfo['module'], pid))
+        else:
+            exist=False
+            logger.info("The module not exist, %s" % _droneInfo['module'])
+        return exist
+
+    def getPid(self, n):
+        _cmd="ssh drone@"+self.droneInfo[n]['host']+" ps -ef | grep "+self.droneInfo[n]['module']+".py | awk '{print $2}' | head -1"
+        print _cmd
+        pid=subprocess.check_output(_cmd, shell=True)
+        if pid!='':
+            pid=int(pid)
+        else:
+            pid = -1
+        return pid
+
+    def getPidAll(self):
+        for i in range(self.nModule):
+            print self.droneInfo[i]['module'], self.getPid(i)
+        
+    def start(self, n):
+        #_droneInfo['pobox'].execRPC(_droneInfo['rpc'], 'Start', {'string':'start', 'times':3}, 'Print', -1.0)
+        self.clear(n)
+
+    def startAll(self):
+        for i in range(self.nModule):
+            self.start(i)
+
+    def configure(self, n):
         pass
 
-    def clear(self):
+    def configureAll(self):
         pass
 
-    def start(self):
-        pass
-    
-    def stop(self):
-        _cmd='pkill -9 -u drone '+self.module
-        stdin, stdout, stderr = self.myssh.exec_command(_cmd)
-        _cmd='pgrep -u drone '+self.module
-        for i in range(10):
-            stdin, stdout, stderr = self.myssh.exec_command(_cmd)
-            print stdin 
-            print stdout 
-            print stderr
-            time.sleep(0.3)
+    def stop(self, n):
+        self.clear(n)
+
+    def stopAll(self):
+        for i in range(self.nModule):
+            self.stop(n)
+
+    def abort(self, n):
+        self.clear(n)
+
+    def abortAll(self):
+        for i in range(self.nModule):
+            self.abort(i)
+
+    def clear(self, n):
+        self.droneInfo[n]['pobox'].execRPC(self.droneInfo[n]['rpc'], 'Clear', {'string':'clear', 'times':3}, 'Print', -1.0)
+        #relist =  _droneInfo['pobox'].getRSLT()
+        #for item in relist:
+
+    def clearAll(self):
+        for i in range(self.nModule):
+            self.clear(i)
+
+    def exit(self):
+        self.killAll()
 
 class setHeartbeat(threading.Thread):
@@ -297,5 +391,5 @@
 
 class getEpicsCommand(threading.Thread):
-    def __init__(self, redisServer, epicsStatus):
+    def __init__(self, redisServer, epicsStatus, droneServer):
         threading.Thread.__init__(self)
         epics.ca.use_initial_context()
@@ -303,4 +397,5 @@
         self.redisServer = redisServer
         self.epicsStatus = epicsStatus
+        self.droneServer = droneServer
 
         self.pvCommand=epics.PV('EXP_IB1_GP:soft:cmd')
@@ -365,7 +460,16 @@
 
     def getDroneStatus(self):
-        droneStatus = self.redisServer.get(self.droneHeartBeat)
-        droneStatus=json.load(droneStatus)['status']
-        droneStatus=json.load(droneStatus)['timestamp']
+        droneStatus=4
+        _statusList=[]
+        try:
+            for i in range(GPPDdroneInfo):
+                _status = self.redisServer.get(self.droneHeartBeat)
+                _statusList.append(json.load(_status)['status'])
+            for i in range(GPPDdroneInfo-1):
+                if _statusList[i+1]!=_statusList[i]:
+                    droneStatus=6
+                    break
+        except:
+            droneStatus=6
         return droneStatus
 
@@ -462,4 +566,16 @@
 
             elif char_value == 1:#START
+                '''
+                _cmd="ssh drone@10.1.33.143 pkill -9 python"
+                _cmd = shlex.split(_cmd)
+                subprocess.Popen(_cmd, shell=False)
+                time.sleep(5.0)            
+                
+                _cmd="ssh drone@10.1.33.143 /home/drone/workspace/drone/daqrun/zhang/pycmd"
+                _cmd = shlex.split(_cmd)
+                subprocess.Popen(_cmd, shell=False)
+                time.sleep(1.0)            
+                '''
+
                 #start
                 _success = self.setCommand(2, 4, 2, 100)
@@ -511,4 +627,13 @@
         logger.error("Redis not available")
     #"==================="
+    # connect Drone
+    try:
+        self.mydrone=getDroneServer(GPPDdroneInfo)
+        self.mydrone.createAll()
+        logger.debug("Drone Created")
+    except:
+        logger.error("Drone Failed")
+
+    #"==================="
     # connect Epics
     try:
@@ -529,6 +654,22 @@
 
     #"==================="
-    try:
-        threadGetEpics = getEpicsCommand(redisServer, epicsStatus)
+    # connect Drone
+    droneServer=None
+    #droneServer=getDroneServer(GPPDdroneInfo)
+    #droneServer.killAll()
+    #droneServer.createAll()
+    #logger.debug("Drone Created")
+    '''
+    try:
+        droneServer=getDroneServer(GPPDdroneInfo)
+        droneServer.killAll()
+        droneServer.createAll()
+        logger.debug("Drone Created")
+    except:
+        logger.error("Drone Failed")
+    '''
+    #"==================="
+    try:
+        threadGetEpics = getEpicsCommand(redisServer, epicsStatus, droneServer)
         threadGetEpics.setDaemon(True)
         threadGetEpics.start()
