Index: BL02auto/BL02auto-v7.py
===================================================================
--- BL02auto/BL02auto-v7.py	(revision ad984afa8e6f72ff7a2e4b00101391a00b1f6f79)
+++ BL02auto/BL02auto-v7.py	(revision ad984afa8e6f72ff7a2e4b00101391a00b1f6f79)
@@ -0,0 +1,1726 @@
+import os
+import sys
+import time
+import glob
+import sqlite3
+import re
+import xml.etree.ElementTree as ET
+from xml.dom import minidom
+import json
+import ssl
+import urllib
+import urllib2
+import subprocess
+import shlex
+import threading
+from datetime import datetime, timedelta
+
+from sqlalchemy import Column, String, create_engine
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import create_engine, Table, Column, Integer, String, MetaData, ForeignKey
+
+ssl._create_default_https_context = ssl._create_unverified_context
+
+Base = declarative_base()
+class record(Base):
+
+    __tablename__ = 'BL02'
+
+    id = Column(Integer, primary_key=True)
+    beamline = Column(String(64), nullable=False)
+    run = Column(String(64), nullable=False, index=True)
+    ctrl = Column(Integer, nullable=False)
+    daq = Column(Integer, nullable=False)
+    monhis = Column(Integer, nullable=False)
+    monevt = Column(Integer, nullable=False)
+    nexus = Column(Integer, nullable=False)
+    begin = Column(String(64), nullable=True)
+    end = Column(String(64), nullable=True)
+    stat = Column(Integer, nullable=False, index=True)
+
+    def __repr__(self):
+        return '%s(%r)' % (self.__class__.__name__, self.run)
+
+CTRLpath="/opt/CSNSDATA/TS1/BL02/Control/"
+DAQpath="/opt/CSNSDATA/TS1/BL02/DAQ/"
+MONHpath="/opt/CSNSDATA/TS1/BL02/Monitor/Histogram/"
+MONEpath="/opt/CSNSDATA/TS1/BL02/Monitor/Event/"
+Nexuspath="/home/transfer/nexus/CSNSDATA/TS1/BL02/"
+softwarePath="/home/transfer/xml/CSNSDATA/TS1/BL02/"
+
+rodsbackup="/csnsZone/home/rods"
+rodsworkspace="/csnsZone/home/public"
+
+global icatTime
+global sessionId, facilityId
+
+#test
+#dmurl="https://dm.csns.ihep.ac.cn:8181"
+dmurl="https://data.csns.ihep.ac.cn"
+
+#=========================================================================================
+def getSessionId():
+    url=dmurl+"/icat/session"
+    header={'content-type': 'application/x-www-form-urlencoded'}
+    body = {'json': '{"plugin":"db", "credentials":[{"username":"dataingestor"}, {"password":"sanlie_123"}]}'}
+    data = urllib.urlencode(body)
+    req = urllib2.Request(url, data, header)
+    res = urllib2.urlopen(req)
+    _tmp=res.read()
+    sessionId=json.loads(_tmp)["sessionId"]
+    return sessionId
+
+def getSessionInfo(sid):
+    try:
+        url = dmurl+"/icat/session/" + sid
+        res = urllib2.urlopen(url)
+        _tmp = res.read()
+        userName = json.loads(_tmp)["userName"]
+    except Exception as ex:
+        print ex
+        userName = None
+
+    return userName
+
+def getFacilityId(sessionId):
+    body="SELECT f.id FROM Facility f WHERE f.name='CSNS'"
+    data = urllib.quote(body)
+    url=dmurl+"/icat/entityManager?sessionId="+sessionId + "&query="+data
+    req = urllib2.Request(url)
+    res = urllib2.urlopen(req)
+    _tmp=res.read()
+    _tmp=json.loads(_tmp)
+    return _tmp[0]
+
+def getInstrumentId(sessionId, name="BL02"):
+    if name=="BL01":
+        nickname="SANS"
+    elif name=="BL02":
+        nickname="RM"
+    elif name=="BL02":
+        nickname="GPPD"
+    else:
+        pass
+
+    body="SELECT f FROM Instrument f WHERE f.facility.name='CSNS'"
+    data = urllib.quote(body)
+    url=dmurl+"/icat/entityManager?sessionId="+sessionId + "&query="+data
+    req = urllib2.Request(url)
+    res = urllib2.urlopen(req)
+    _tmp=res.read()
+    
+    _tmp=json.loads(_tmp)
+
+    _ins=[]
+    for f in _tmp:
+        _ins.append(f['Instrument'])
+
+    for f in _ins:
+        if f['name']==name or f['name']==nickname:
+            _id=f['id']
+            break
+    return _id
+
+def getInvestigationId(sessionId, investigationName, visitName):
+    body="SELECT f FROM Investigation f WHERE f.facility.name='CSNS' and f.name="+"'"+investigationName+"' and f.visitId="+"'"+visitName+"'"
+    data = urllib.quote(body)
+    url=dmurl+"/icat/entityManager?sessionId="+sessionId + "&query="+data
+    req = urllib2.Request(url)
+    res = urllib2.urlopen(req)
+    _tmp=res.read()
+
+    _tmp=json.loads(_tmp)[0]
+    return _tmp['Investigation']['id']
+
+def getInvestigationDate(investigationId):
+    body="SELECT f FROM Investigation f WHERE f.id="+"'"+investigationId+"'"
+    data = urllib.quote(body)
+    url=dmurl+"/icat/entityManager?sessionId="+sessionId + "&query="+data
+    req = urllib2.Request(url)
+    res = urllib2.urlopen(req)
+    _tmp=res.read()
+
+    _tmp=json.loads(_tmp)[0]
+    return _tmp['Investigation']['startDate'], _tmp['Investigation']['endDate']
+
+def setSample(sessionId, investigationId, sampleTypeId, sampleName="unknown"):
+    url=dmurl+"/icat/entityManager"
+    header={'content-type': 'application/x-www-form-urlencoded'}
+
+    entities='[{"Sample":{"type":{"id":sampleTypeId},"investigation":{"id":investigationId},"name":"sampleName"}}]'
+
+    entities =re.sub('sampleTypeId', str(sampleTypeId), entities)
+    entities =re.sub('investigationId', str(investigationId), entities)
+    entities =re.sub('sampleName', sampleName, entities)
+
+    body={}
+    body['sessionId']=sessionId
+    body['entities'] = entities
+
+    data = urllib.urlencode(body)
+    data=re.sub('\+', '%20', data)
+    req = urllib2.Request(url, data, header)
+    res = urllib2.urlopen(req)
+    _tmp=res.read()
+
+    _tmp=json.loads(_tmp)
+    return  _tmp[0]
+
+
+def setSampleType(sessionId, facilityId, sampleTypeName="unknown", molecularFormula="unknown"):
+    url=dmurl+"/icat/entityManager"
+    header={'content-type': 'application/x-www-form-urlencoded'}
+
+    entities='[{"SampleType":{"facility":{"id":facilityId},"molecularFormula":"molecularFormulaName","name":"sampleTypeName"}}]'
+    
+    entities =re.sub('facilityId', str(facilityId), entities)
+    entities =re.sub('molecularFormulaName', molecularFormula, entities)
+    entities =re.sub('sampleTypeName', sampleTypeName, entities)
+
+    body={}
+    body['sessionId']=sessionId
+    body['entities'] = entities
+
+    data = urllib.urlencode(body)
+    data=re.sub('\+', '%20', data)
+    req = urllib2.Request(url, data, header)
+    res = urllib2.urlopen(req)
+    _tmp=res.read()
+
+    _tmp=json.loads(_tmp)
+    return  _tmp[0]
+
+def getSampleTypeId(sessionId, sampleTypeName):
+    body = "SELECT f FROM SampleType f WHERE f.facility.name='CSNS' and f.name="+"'"+sampleTypeName+"'"
+    data = urllib.quote(body)
+    url=dmurl+"/icat/entityManager?sessionId="+sessionId + "&query="+data
+    req = urllib2.Request(url)
+    res = urllib2.urlopen(req)
+    _tmp=res.read()
+
+    _tmp=json.loads(_tmp)[0]
+    return _tmp['SampleType']['id']
+
+def getSampleId(sessionId, investigationId, sampleName):
+    body = "SELECT f FROM Sample f WHERE f.investigation.id="+investigationId+" and f.name="+"'"+sampleName+"'"
+    data = urllib.quote(body)
+    url=dmurl+"/icat/entityManager?sessionId="+sessionId + "&query="+data
+
+    req = urllib2.Request(url)
+    res = urllib2.urlopen(req)
+    _tmp=res.read()
+
+    _tmp=json.loads(_tmp)[0]
+    return _tmp['Sample']['id']
+
+def getDatasetTypeId(sessionId, name):
+    body = "SELECT f FROM DatasetType f WHERE f.facility.name='CSNS' and f.name="+"'"+name+"'"
+    data = urllib.quote(body)
+    url=dmurl+"/icat/entityManager?sessionId="+sessionId + "&query="+data
+    req = urllib2.Request(url)
+    res = urllib2.urlopen(req)
+    _tmp=res.read()
+
+    _tmp=json.loads(_tmp)[0]
+    return _tmp['DatasetType']['id']
+
+def getDatasetId(sessionId, investigationId, datasetName):
+    body = "SELECT f FROM Dataset f WHERE f.investigation.id="+investigationId+" and f.name="+"'"+datasetName+"'"
+    data = urllib.quote(body)
+    url=dmurl+"/icat/entityManager?sessionId="+sessionId + "&query="+data
+    req = urllib2.Request(url)
+    res = urllib2.urlopen(req)
+    _tmp=res.read()
+
+    _tmp=json.loads(_tmp)[0]
+    return _tmp['Dataset']['id']
+
+def setDataset(sessionId, investigationId, datasetTypeId, sampleId, datasetName, dataLocation, startDatetime, endDatetime, dataDescription):
+    url=dmurl+"/icat/entityManager"
+    header={'content-type': 'application/x-www-form-urlencoded'}
+
+    if startDatetime=="unknown" or endDatetime=="unknown":
+        entities='[{"Dataset":{"investigation":{"id":investigationId},"sample":{"id":sampleId},"type":{"id":datasetTypeId},"complete":true,"location":"dataLocation", "name":"datasetName","description":"dataDescription"}}]'
+    else:
+        entities='[{"Dataset":{"investigation":{"id":investigationId},"sample":{"id":sampleId},"type":{"id":datasetTypeId},"complete":true,"location":"dataLocation", "name":"datasetName","startDate":"startDatetime","endDate":"endDatetime","description":"dataDescription"}}]'
+
+    entities =re.sub('investigationId', str(investigationId), entities)
+    entities =re.sub('datasetTypeId', str(datasetTypeId), entities)
+    entities =re.sub('sampleId', str(sampleId), entities)
+    entities =re.sub('datasetName', str(datasetName), entities)
+    entities =re.sub('dataLocation', str(dataLocation), entities)
+    entities =re.sub('startDatetime', str(startDatetime), entities)
+    entities =re.sub('endDatetime', str(endDatetime), entities)
+    entities =re.sub('dataDescription', str(dataDescription), entities)
+
+    body={}
+    body['sessionId']=sessionId
+    body['entities'] = entities
+
+    data = urllib.urlencode(body)
+    data=re.sub('\+', '%20', data)
+
+    req = urllib2.Request(url, data, header)
+    res = urllib2.urlopen(req)
+    _tmp=res.read()
+
+    _tmp=json.loads(_tmp)
+    return  _tmp[0]
+
+def getDatafileFormatId(sessionId, datafileFormatName, datafileFormatVersion):
+    body="SELECT f FROM DatafileFormat f WHERE f.facility.name='CSNS' and f.name="+"'"+datafileFormatName+"' and f.version="+"'"+datafileFormatVersion+"'"
+    data = urllib.quote(body)
+    url=dmurl+"/icat/entityManager?sessionId="+sessionId + "&query="+data
+    req = urllib2.Request(url)
+    res = urllib2.urlopen(req)
+    _tmp=res.read()
+
+    _tmp=json.loads(_tmp)[0]
+    return _tmp['DatafileFormat']['id']
+
+def setDatafile(sessionId, datasetId, datafileFormatId, datafileName, dataCreateTime, dataDescription, dataSize, dataLocation):
+    url=dmurl+"/icat/entityManager"
+    header={'content-type': 'application/x-www-form-urlencoded'}
+
+    entities='[{"Datafile":{"dataset":{"id":datasetId},"datafileFormat":{"id":datafileFormatId},"datafileCreateTime":"dataCreateTime","description":"dataDescription","fileSize":dataSize,"location":"dataLocation","name":"datafileName"}}]'
+
+    entities =re.sub('datasetId', str(datasetId), entities)
+    entities =re.sub('datafileFormatId', str(datafileFormatId), entities)
+    entities =re.sub('datafileName', str(datafileName), entities)
+    entities =re.sub('dataCreateTime', str(dataCreateTime), entities)
+    entities =re.sub('dataDescription', str(dataDescription), entities)
+    entities =re.sub('dataSize', str(dataSize), entities)
+    entities =re.sub('dataLocation', str(dataLocation), entities)
+
+    body={}
+    body['sessionId']=sessionId
+    body['entities'] = entities
+
+    data = urllib.urlencode(body)
+    data=re.sub('\+', '%20', data)
+    req = urllib2.Request(url, data, header)
+    res = urllib2.urlopen(req)
+    _tmp=res.read()
+
+    _tmp=json.loads(_tmp)
+    return  _tmp[0]
+
+#=========================================================================================
+
+def getCreationDate(_file):
+    stat = os.stat(_file)
+    try:
+        _tmp=stat.st_birthtime
+    except AttributeError:
+        _tmp=stat.st_ctime
+        #_tmp=stat.st_mtime
+
+    #return time.strftime('%Y-%m-%dT%H:%M:%S%z', time.localtime(int(_tmp)))
+    return time.strftime('%Y-%m-%dT%H:%M:%S.000+08:00', time.localtime(int(_tmp)))
+
+def getFileSize(_file):
+    #unit: byte
+    #return os.path.getsize(_file)
+    return int(os.stat(_file).st_size)
+
+def getFileName(_file):
+    return os.path.basename(_file)
+#=========================================================================================
+
+def getRunNoList(dir='.'):
+    pattern="^RUN[0-9]{7}"
+    runNo=[]
+
+    for dirpath, dirnames, files in os.walk(dir):
+        for dirname in dirnames:
+            if re.match(pattern, dirname):
+                runNo.append(int(dirname[3:]))
+    
+    runNo.sort()           
+    
+    return runNo
+
+def getControlPath(runNo):
+    return CTRLpath+"RUN"+str(runNo).zfill(7)+"/"
+
+def getDetectorPath(runNo):
+    _tmp1=DAQpath+"RUN"+str(runNo).zfill(7)+"/"
+    _tmp2=DAQpath+"RUN"+str(runNo)+"/"
+    if os.path.exists(_tmp1):
+        _file=_tmp1
+    else:
+        if os.path.exists(_tmp2):
+            _file=_tmp2
+        else:
+            _file= None
+
+    return _file
+
+def getDetectorFiles(runNo):
+    _path=getDetectorPath(runNo)
+
+    _files=[]
+    try:
+        _file1=glob.glob(_path+"*.xml")
+        _file2=glob.glob(_path+"*.dat")
+        _files=_file1 + _file2
+    except:
+        pass
+
+    filelist=[]
+    for f in _files:
+        _filename=getFileName(f)
+        _filetime=getCreationDate(f)
+        _filesize=getFileSize(f)
+        _fromlocation=f
+        _tolocation="/raw/CSNS/TS1/BL02/"+getRunString(runNo)+"/Detector/"+_filename
+        _dataDescription="detector-raw"
+        _datafileFormatName="raw_detector"    
+        #for test
+        #_datafileFormatName="dat"    
+        filelist.append([_filename, _filetime, str(_filesize), _fromlocation, _tolocation, _dataDescription, _datafileFormatName])
+    return filelist
+
+def getDAQFileList(runNo):
+    #ns={'d': 'http://csns.ihep.ac.cn/xml/ns/descriptor'}
+    path=getDetectorPath(runNo)
+
+    try:
+        _file = glob.glob(path+"CSNS-TS1-BL02-DAQ-summary-*.xml")
+        _file=_file[-1]
+    except Exception as ex:
+	print ex
+        return
+
+    with open(_file) as f:
+        xmlstring = f.read()
+    
+    if 'descriptor' in xmlstring: 
+        #xmlstring=re.sub(' xmlns="[^"]+"', '', xmlstring, count=1)
+        xmlstring=re.sub('\<daqDescriptor.+', '<root>', xmlstring)
+        xmlstring=re.sub('\<\/daqDescriptor\>', '</root>', xmlstring, count=1)
+    else:
+        xmlstring = re.sub('\<\?xml version="1.0" encoding="UTF-8"\?\>', '<?xml version="1.0" encoding="UTF-8"?>\n <root>', xmlstring)
+        xmlstring +="</root>"
+
+    _files=[]
+
+    try:
+        root = ET.fromstring(xmlstring)
+        _modules=root.findall("module")
+        for a in _modules:
+            #print a.attrib['id']
+            _child=a.getchildren()[0].find(".//file")
+            for _f in _child:
+                _files.append(_f.text)
+    except Exception as ex:
+	print ex
+        pass
+
+    return _files
+
+def getDetectorSummaryFile(runNo):
+    path=getDetectorPath(runNo)
+
+    try:
+        _file = glob.glob(path+"CSNS-TS1-BL02-DAQ-summary-*.xml")
+        _file=_file[-1]
+    except Exception as ex:
+	print ex
+        _file=None
+
+    return _file
+
+def getDetectorData(runNo):
+    path=getDetectorPath(runNo)
+
+    try:
+        _file = glob.glob(path+"BL02*.dat")
+    except:
+        _file=None
+
+    return _file
+
+def hasDetectorPath(runNo):
+    _tmp1=DAQpath+"RUN"+str(runNo).zfill(7)+"/"
+    _tmp2=DAQpath+"RUN"+str(runNo)+"/"
+    if os.path.exists(_tmp1):
+        _result=1
+    else:
+        if os.path.exists(_tmp2):
+            _result=1
+        else:
+            _result= 0
+
+    return _result
+
+def hasNexusPath(runNo):
+    _tmp=Nexuspath+getRunString(runNo)+"/"
+    if os.path.exists(_tmp):
+        _result= 1
+    else:
+        _result= 0
+
+    return _result
+
+def getMONhisPath(runNo):
+    _path="/opt/CSNSDATA/TS1/BL02/Monitor/Histogram/"
+    _tmp=_path+"RUN"+str(runNo).zfill(7)+"/"
+    if os.path.exists(_tmp):
+        _file=_tmp
+    else:
+        _file=None
+    return _file
+
+def hasMonitorHisPath(runNo):
+    _path="/opt/CSNSDATA/TS1/BL02/Monitor/Histogram/"+"RUN"+str(runNo).zfill(7)+"/"
+
+    if os.path.exists(_path):
+        _result= 1
+    else:
+        _result= 0
+    
+    return _result
+
+def getMonitorhisFiles(runNo):
+    _path=getMONhisPath(runNo)
+
+    _files=[]
+    try:
+        _file1=glob.glob(_path+"*.bin")
+        _files=_file1
+    except:
+        pass
+
+    filelist=[]
+    for f in _files:
+        _filename=getFileName(f)
+        if len(_filename)==34:
+            _fileDBname=_filename[0:16]+"M"+str(i)+"_"+_filename[16:]
+        else:
+            _fileDBname=_filename
+
+        _filetime=getCreationDate(f)
+        _filesize=getFileSize(f)
+        _fromlocation=f
+        _tolocation="/raw/CSNS/TS1/BL02/"+getRunString(runNo)+"/Monitor"+str(i)+"/Histogram/"+_filename
+        _dataDescription="monitor-histogram"
+        _datafileFormatName="raw_monitor"
+        # for test
+        #_datafileFormatName="dat"
+        filelist.append([_fileDBname, _filetime, str(_filesize), _fromlocation, _tolocation, _dataDescription, _datafileFormatName])
+    return filelist
+
+def getMONevtPath(runNo):
+    _path="/opt/CSNSDATA/TS1/BL02/Monitor/Event/"
+    _tmp=_path+"RUN"+str(runNo).zfill(7)+"/"
+    if os.path.exists(_tmp):
+        _file=_tmp
+    else:
+        _file=None
+
+    return _file
+
+def hasMonitorEvtPath(runNo):
+    _path="/opt/CSNSDATA/TS1/BL02/Monitor/Event/"+"RUN"+str(runNo).zfill(7)+"/"
+
+    if os.path.exists(_path):
+        _result= 1
+    else:
+        _result= 0
+
+    return _result
+
+def getMonitorevtFiles(runNo):
+    _path=getMONevtPath(runNo)
+
+    _files=[]
+    try:
+        _file1=glob.glob(_path+"*.h5")
+        _files=_file1
+    except:
+        pass
+
+    filelist=[]
+    for f in _files:
+        _filename=getFileName(f)
+        if len(_filename)==41:
+            _fileDBname=_filename[0:16]+"M"+str(i)+"_"+_filename[16:]
+        else:
+            _fileDBname=_filename
+
+        _filetime=getCreationDate(f)
+        _filesize=getFileSize(f)
+        _fromlocation=f
+        _tolocation="/raw/CSNS/TS1/BL02/"+getRunString(runNo)+"/Monitor"+str(i)+"/Event/"+_filename
+        _dataDescription="monitor-nexus"
+        _datafileFormatName="nexus_monitor"
+ 
+        filelist.append([_fileDBname, _filetime, str(_filesize), _fromlocation, _tolocation, _dataDescription, _datafileFormatName])
+    return filelist
+
+def getNexusPath(runNo):
+    runString=getRunString(runNo)
+    return "/home/transfer/nexus/CSNSDATA/TS1/BL02/"+runString+"/"
+
+def getSoftwarePath(runNo):
+    runString=getRunString(runNo)
+    return "/home/transfer/xml/CSNSDATA/TS1/BL02/"+runString+"/"
+
+def getNexusFiles(runNo):
+    _path=getNexusPath(runNo)
+    
+    _files=[]
+    try:
+        _file1=glob.glob(_path+"*.nxs")
+        _files=_file1
+    except:
+        pass
+
+    filelist=[]
+    for f in _files:
+        _filename=getFileName(f)
+        _filetime=getCreationDate(f)
+        _filesize=getFileSize(f)
+        _fromlocation=f
+        _tolocation="/nexus/CSNS/TS1/BL02/" +getRunString(runNo)+"/"+_filename
+        _dataDescription="nexus"
+        _datafileFormatName="nexus"
+        # for test
+        #_datafileFormatName="nxs"
+    
+        filelist.append([_filename, _filetime, str(_filesize), _fromlocation, _tolocation, _dataDescription, _datafileFormatName])
+    
+    return filelist
+
+def getSoftwareFiles(runNo):
+    _path=getSoftwarePath(runNo)
+
+    _files=[]
+    try:
+        _file1=glob.glob(_path+"*.xml")
+        _files=_file1
+    except:
+        pass
+
+    filelist=[]
+    for f in _files:
+        _filename=getFileName(f)
+        _filetime=getCreationDate(f)
+        _filesize=getFileSize(f)
+        _fromlocation=f
+        _tolocation="/raw/CSNS/TS1/BL02/" +getRunString(runNo)+"/Software/"+_filename
+        _dataDescription="software summary"
+        _datafileFormatName="software_summary"
+        # for test
+        #_datafileFormatName="xml"
+
+        filelist.append([_filename, _filetime, str(_filesize), _fromlocation, _tolocation, _dataDescription, _datafileFormatName])
+
+    return filelist
+
+def getControlSummaryFile(runNo):
+    _path=getControlPath(runNo)
+    _files = glob.glob(_path+"CSNS-TS1-BL02-CTRL-Summary-*.xml")
+    if _files:
+        _file=_files[0]
+    else:
+        _file=None
+    return _file
+
+def getControlFiles(runNo):
+    _path=getControlPath(runNo)
+
+    _files=[]
+    try:
+        _file1=glob.glob(_path+"*.xml")
+        _files=_file1
+    except Exception as ex:
+	print ex
+        pass
+
+    filelist=[]
+    for f in _files:
+        _filename=getFileName(f)
+        _filetime=getCreationDate(f)
+        _filesize=getFileSize(f)
+        _fromlocation=f
+        _tolocation="/raw/CSNS/TS1/BL02/"+getRunString(runNo)+"/Control/"+_filename
+        _dataDescription="control-raw"
+        _datafileFormatName="control_summary"
+        # for test
+        #_datafileFormatName="xml"
+    
+        filelist.append([_filename, _filetime, str(_filesize), _fromlocation, _tolocation, _dataDescription, _datafileFormatName])
+    return filelist
+
+def hasControlSummaryFile(runNo):
+    _path=getControlPath(runNo)
+    _files = glob.glob(_path+"CSNS-TS1-BL02-CTRL-Summary-*.xml")
+    if _files:
+        _result= 1
+    else:
+        _result= 0
+    return _result
+
+def getRunTime(runNo):
+    _file=getControlSummaryFile(runNo)
+
+    tree = ET.parse(_file)
+    root = tree.getroot()
+
+    _tmp = root.findall("NXentry/start_time_utc")[0].text
+    if _tmp:
+        _tmp=re.sub(' ', 'T', _tmp)
+        _tmp+=".000+08:00"
+        beginDate=_tmp
+    else:
+        beginData=""
+
+    _tmp = root.findall('NXentry/end_time_utc')[0].text
+    if _tmp:
+        _tmp=re.sub(' ', 'T', _tmp)
+        _tmp+=".000+08:00"
+        endDate=_tmp
+    else:
+        endData=""
+
+    return beginDate, endDate
+
+#0: check done
+#1: nexus begin
+#2: nexus done
+#3: irods begin
+#4: irods done
+#5: icat begin
+#6: icat done
+
+def getIcatInfo():
+    sessionId = getSessionId()
+    facilityId = getFacilityId(sessionId)
+    return sessionId, facilityId
+
+def setIcatDataset(sessionId, facilityId, investigationId, sampleTypeName, sampleName, molecularFormula, datasetName, dataLocation, startDatetime, endDatetime, dataDescription):
+    try:
+        setSampleType(sessionId, facilityId, sampleTypeName, molecularFormula)
+    except Exception as ex:
+	print ex
+        pass
+        
+    sampleTypeId = getSampleTypeId(sessionId, sampleTypeName)
+    #print "get sampleTypeId: ", sampleTypeId
+
+    try:
+        setSample(sessionId, investigationId, sampleTypeId, sampleName)
+        #print "set sampleId: ", sampleId
+    except Exception as ex:
+	print ex
+        pass
+
+    sampleId =  getSampleId(sessionId, str(investigationId), sampleName)
+
+    try:
+        investigationSampleId= setInvestigationSample(sessionId, investigationId, sampleTypeId, sampleName)
+        #print "set investigationSampleId: ", investigationSampleId
+    except Exception as ex:
+	print ex
+        pass
+    
+
+    datasetTypeId= getDatasetTypeId(sessionId, "raw")
+    #print "get datasetTypeId: ", datasetTypeId
+
+    try:
+        setDataset(sessionId, investigationId, datasetTypeId, sampleId, datasetName, dataLocation, startDatetime, endDatetime, dataDescription)
+    except Exception as ex:
+	print ex
+        pass
+
+    datasetId = getDatasetId(sessionId, str(investigationId), datasetName) 
+    #print "set datasetId: ", datasetId
+    
+    return datasetId
+
+def setIcatDatafile(sessionId, datasetId, datafileFormatName, datafileName, dataCreateTime, dataDescription, dataSize, dataLocation):
+    try:
+        datafileFormatId = getDatafileFormatId(sessionId, datafileFormatName, datafileFormatVersion="1.0")
+        print "datafileFormatId: ", datafileFormatId
+
+        result = setDatafile(sessionId, datasetId, datafileFormatId, datafileName, dataCreateTime, dataDescription, dataSize, dataLocation)
+        print " set datafile: ", result
+    except Exception as ex:
+	print ex
+        pass
+
+def setInvestigationSample(sessionId, investigationId, sampleTypeId, sampleName):
+    url=dmurl+"/icat/entityManager"
+    header={'content-type': 'application/x-www-form-urlencoded'}
+
+    entities='[{"Sample":{"type":{"id":sampleTypeId},"investigation":{"id":investigationId},"name":"sampleName"}}]'
+
+    entities =re.sub('investigationId', str(investigationId), entities)
+    entities =re.sub('sampleTypeId', str(sampleTypeId), entities)
+    entities =re.sub('sampleName', str(sampleName), entities)
+
+    body={}
+    body['sessionId']=sessionId
+    body['entities'] = entities
+
+    data = urllib.urlencode(body)
+    data=re.sub('\+', '%20', data)
+    req = urllib2.Request(url, data, header)
+    res = urllib2.urlopen(req)
+    _tmp=res.read()
+
+    _tmp=json.loads(_tmp)
+    return  _tmp[0]
+
+
+def writeXml(runNo, beginTime, endTime, controlFiles, detectorFiles, monitorhisFiles, monitorevtFiles, nexusFiles):
+    #from ElementTree_pretty import prettify
+
+    runString = getRunString(runNo)
+    _tmp="mkdir -p "+getSoftwarePath(runNo)
+    subprocess.check_output(_tmp, shell=True)
+
+
+    top = ET.Element('root')
+
+    child = ET.SubElement(top, 'run')
+    child.text=runString
+
+    child = ET.SubElement(top, 'begin')
+    child.text=beginTime
+
+    child = ET.SubElement(top, 'end')
+    child.text=endTime
+
+    _num=len(controlFiles)
+    control = ET.SubElement(top, 'contorl')
+    control.set('number',str(_num))
+    for filename, filesize, filecreation, fromlocation, tolocation, _,_ in controlFiles:
+        _file = ET.SubElement(control, 'file')
+        _child = ET.SubElement(_file, 'filename')
+        _child.text=filename
+        
+        _child = ET.SubElement(_file, 'filesize')
+        _child.text=filesize
+    
+        _child = ET.SubElement(_file, 'creationtime')
+        _child.text=filecreation
+
+        _child = ET.SubElement(_file, 'from')
+        _child.text=fromlocation
+
+        _child = ET.SubElement(_file, 'to')
+        _child.text=tolocation
+
+    _num=len(detectorFiles)
+    detector = ET.SubElement(top, 'detector')
+    detector.set('number',str(_num))
+    for filename, filesize, filecreation, fromlocation, tolocation, _,_ in detectorFiles:
+        _file = ET.SubElement(detector, 'file')
+        _child = ET.SubElement(_file, 'filename')
+        _child.text=filename
+
+        _child = ET.SubElement(_file, 'filesize')
+        _child.text=filesize
+
+        _child = ET.SubElement(_file, 'creationtime')
+        _child.text=filecreation
+
+        _child = ET.SubElement(_file, 'from')
+        _child.text=fromlocation
+
+        _child = ET.SubElement(_file, 'to')
+        _child.text=tolocation
+        
+    _num=len(monitorhisFiles)
+    _monitor = ET.SubElement(top, "monitor_histogram")
+    _monitor.set('number',str(_num))
+    for filename, filesize, filecreation, fromlocation, tolocation, _,_ in monitorhisFiles:
+        _file = ET.SubElement(_monitor, 'file')
+        _child = ET.SubElement(_file, 'filename')
+        _child.text=filename
+
+        _child = ET.SubElement(_file, 'filesize')
+        _child.text=filesize
+
+        _child = ET.SubElement(_file, 'creationtime')
+        _child.text=filecreation
+
+        _child = ET.SubElement(_file, 'from')
+        _child.text=fromlocation
+            
+        _child = ET.SubElement(_file, 'to')
+        _child.text=tolocation
+
+     
+    _num=len(monitorevtFiles)
+    _monitor = ET.SubElement(top, "monitor_event")
+    _monitor.set('number',str(_num))
+    for filename, filesize, filecreation, fromlocation, tolocation, _,_ in monitorevtFiles:
+        _file = ET.SubElement(_monitor, 'file')
+        _child = ET.SubElement(_file, 'filename')
+        _child.text=filename
+
+        _child = ET.SubElement(_file, 'filesize')
+        _child.text=filesize
+
+        _child = ET.SubElement(_file, 'creationtime')
+        _child.text=filecreation
+
+        _child = ET.SubElement(_file, 'from')
+        _child.text=fromlocation
+            
+        _child = ET.SubElement(_file, 'to')
+        _child.text=tolocation
+
+    _num=len(nexusFiles)
+    nexus = ET.SubElement(top, 'nexus')
+    nexus.set('number',str(_num))
+    for filename, filesize, filecreation, fromlocation, tolocation, _,_ in nexusFiles:
+        _file = ET.SubElement(nexus, 'file')
+        _child = ET.SubElement(_file, 'filename')
+        _child.text=filename
+
+        _child = ET.SubElement(_file, 'filesize')
+        _child.text=filesize
+
+        _child = ET.SubElement(_file, 'creationtime')
+        _child.text=filecreation
+
+        _child = ET.SubElement(_file, 'from')
+        _child.text=fromlocation
+
+        _child = ET.SubElement(_file, 'to')
+        _child.text=tolocation
+
+    xmlstr = minidom.parseString(ET.tostring(top)).toprettyxml(indent="   ")
+    with open(getSoftwarePath(runNo)+runString+".xml", "w") as f:
+        f.write(xmlstr)
+
+def setControlLocation(runNo, _filename):
+    backup="/raw/CSNS/TS1/BL02/"+getRunString(runNo)+"/Control/"+_filename
+    return backup
+    
+def setDetectorLocation(runNo, _filename):
+    backup="/raw/CSNS/TS1/BL02/"+getRunString(runNo)+"/Detector/"+_filename
+    return backup
+    
+def setMonitorhisLocation(runNo, k, _filename):
+    backup="/raw/CSNS/TS1/BL02/"+getRunString(runNo)+"/Monitor"+str(k)+"/Histogram/"+_filename
+    return backup
+    
+def setMonitorevtLocation(runNo, k, _filename):
+    backup="/raw/CSNS/TS1/BL02/"+getRunString(runNo)+"/Monitor"+str(k)+"/Event/"+_filename
+    return backup
+    
+def setNexusLocation(runNo, k, _filename):
+    backup="/nexus/CSNS/TS1/BL02/" +getRunString(runNo)+"/"+_filename
+    return backup
+
+def getRunString(runNo):
+    return "RUN"+str(runNo).zfill(7)
+
+def writeNexusToIcat(runNo, conf, investigationId):
+    global icatTime
+
+    global sessionId, facilityId
+
+    if time.time()-icatTime>5400:
+        icatTime=time.time()
+        sessionId, facilityId = getIcatInfo()
+
+    datasetId =setIcatDataset(sessionId, facilityId, investigationId, conf['sampleTypeName'], conf['sampleName'], conf['sampleTypeMolecularFormula'], conf['datasetName'], conf['dataLocation'], conf['startDatetime'], conf['endDatetime'], conf['datasetDescription'])
+
+    _nexusFiles       = getNexusFiles(runNo)
+
+    for datafileName, dataCreateTime, dataSize,  _, dataLocation, dataDescription, datafileFormatName  in _nexusFiles:
+        setIcatDatafile(sessionId, datasetId, datafileFormatName, datafileName, dataCreateTime, dataDescription, dataSize, dataLocation)
+
+    return True
+
+def writeRawToIcat(runNo, conf, investigationId):
+    global icatTime
+
+    global sessionId, facilityId
+
+    if time.time()-icatTime>5400:
+        icatTime=time.time()
+        sessionId, facilityId = getIcatInfo()
+
+    datasetId =setIcatDataset(sessionId, facilityId, investigationId, conf['sampleTypeName'], conf['sampleName'], conf['sampleTypeMolecularFormula'], conf['datasetName'], conf['dataLocation'], conf['startDatetime'], conf['endDatetime'], conf['datasetDescription'])
+
+    _controlFiles=getControlFiles(runNo)
+    _detectorFiles=getDetectorFiles(runNo)
+    _monitorhisFiles = getMonitorhisFiles(runNo)
+    _monitorevtFiles = getMonitorevtFiles(runNo)
+    _nexusFiles       = getNexusFiles(runNo)
+    _softwareFiles    = getSoftwareFiles(runNo)
+
+    for datafileName, dataCreateTime, dataSize,  _, dataLocation, dataDescription, datafileFormatName  in _controlFiles:
+        setIcatDatafile(sessionId, datasetId, datafileFormatName, datafileName, dataCreateTime, dataDescription, dataSize, dataLocation)
+
+    for datafileName, dataCreateTime, dataSize,  _, dataLocation, dataDescription, datafileFormatName  in _detectorFiles:
+        setIcatDatafile(sessionId, datasetId, datafileFormatName, datafileName, dataCreateTime, dataDescription, dataSize, dataLocation)
+
+    for datafileName, dataCreateTime, dataSize,  _, dataLocation, dataDescription, datafileFormatName  in _monitorhisFiles:
+        setIcatDatafile(sessionId, datasetId, datafileFormatName, datafileName, dataCreateTime, dataDescription, dataSize, dataLocation)
+
+    for datafileName, dataCreateTime, dataSize,  _, dataLocation, dataDescription, datafileFormatName  in _monitorevtFiles:
+        setIcatDatafile(sessionId, datasetId, datafileFormatName, datafileName, dataCreateTime, dataDescription, dataSize, dataLocation)
+
+    for datafileName, dataCreateTime, dataSize,  _, dataLocation, dataDescription, datafileFormatName  in _nexusFiles:
+        setIcatDatafile(sessionId, datasetId, datafileFormatName, datafileName, dataCreateTime, dataDescription, dataSize, dataLocation)
+
+    for datafileName, dataCreateTime, dataSize,  _, dataLocation, dataDescription, datafileFormatName  in _softwareFiles:
+        setIcatDatafile(sessionId, datasetId, datafileFormatName, datafileName, dataCreateTime, dataDescription, dataSize, dataLocation)
+
+    return True
+
+def setInvestigationDate(conf, investigationId):
+    global sessionId
+
+    try:
+        oldStartDate, oldEndDate= getInvestigationDate(investigationId)
+    except:
+        oldStartDate=""
+        oldEndDate=""
+
+    newStartDate = conf['startDatetime']
+    newEndDate = conf['endDatetime']
+
+    boolStart=False
+    startDate=oldStartDate
+    if newStartDate!="unknown":
+        _newDate = datetime.strptime(newStartDate, '%Y-%m-%dT%H:%M:%S.%f+08:00')
+        if oldStartDate=="":
+            startDate=newStartDate
+            boolStart=True
+        else:
+            _oldDate = datetime.strptime(oldStartDate, '%Y-%m-%dT%H:%M:%S.%f+08:00')
+            if (_oldDate-_newDate) > timedelta(seconds=1):
+                startDate=newStartDate
+                boolStart=True
+
+    boolEnd=False
+    endDate=oldEndDate
+    if newEndDate!="unknown":
+        _newDate = datetime.strptime(newEndDate, '%Y-%m-%dT%H:%M:%S.%f+08:00')
+        if oldEndDate=="":
+            endDate=newEndDate
+            boolEnd=True
+        else:
+            _oldDate = datetime.strptime(oldEndDate, '%Y-%m-%dT%H:%M:%S.%f+08:00')
+            if (_newDate-_oldDate) > timedelta(seconds=1):
+                endDate=newEndDate
+                boolEnd=True
+
+    url=dmurl+"/icat/entityManager"
+    header={'content-type': 'application/x-www-form-urlencoded'}
+
+    if boolStart and boolEnd:
+        entities='[{"Investigation":{"id":investigationId, "startDate":"startDateTime", "endDate":"endDateTime"}}]'
+    if boolStart and not boolEnd:
+        entities='[{"Investigation":{"id":investigationId, "startDate":"startDateTime"}}]'
+    if not boolStart and boolEnd:
+        entities='[{"Investigation":{"id":investigationId, "endDate":"endDateTime"}}]'
+    if not boolStart and not boolEnd:
+        entities='[{"Investigation":{"id":investigationId}}]'
+
+    entities =re.sub('investigationId', str(investigationId), entities)
+    entities =re.sub('startDateTime', str(startDate), entities)
+    entities =re.sub('endDateTime', str(endDate), entities)
+
+    body={}
+    body['sessionId']=sessionId
+    body['entities'] = entities
+
+    data = urllib.urlencode(body)
+    data=re.sub('\+', '%20', data)
+    req = urllib2.Request(url, data, header)
+    res = urllib2.urlopen(req)
+    _tmp=res.read()
+
+    _tmp=json.loads(_tmp)
+    #return  _tmp[0]
+    return  _tmp
+
+def mkdirRods(runNo):
+    runString=getRunString(runNo)
+    _tmp = "imkdir -p "+ "/csnsZone/home/rods/raw/CSNS/TS1/BL02/"+runString+"/Control"
+    subprocess.check_output(_tmp, shell=True)
+    _tmp = "imkdir -p "+ "/csnsZone/home/rods/raw/CSNS/TS1/BL02/"+runString+"/Detector"
+    subprocess.check_output(_tmp, shell=True)
+    _tmp = "imkdir -p "+ "/csnsZone/home/rods/raw/CSNS/TS1/BL02/"+runString+"/Monitor/Histogram"
+    subprocess.check_output(_tmp, shell=True)
+    _tmp = "imkdir -p "+ "/csnsZone/home/rods/raw/CSNS/TS1/BL02/"+runString+"/Monitor/Event"
+    subprocess.check_output(_tmp, shell=True)
+    _tmp = "imkdir -p "+ "/csnsZone/home/rods/raw/CSNS/TS1/BL02/"+runString+"/Software"
+    subprocess.check_output(_tmp, shell=True)
+    _tmp = "imkdir -p "+ "/csnsZone/home/rods/nexus/CSNS/TS1/BL02/"+runString
+    subprocess.check_output(_tmp, shell=True) 
+
+def mkdirPublic(runNo):
+    runString=getRunString(runNo)
+    _tmp = "imkdir -p "+ "/csnsZone/home/public/raw/CSNS/TS1/BL02/"+runString+"/Control"
+    subprocess.check_output(_tmp, shell=True)
+    _tmp = "imkdir -p "+ "/csnsZone/home/public/raw/CSNS/TS1/BL02/"+runString+"/Detector"
+    subprocess.check_output(_tmp, shell=True)
+    _tmp = "imkdir -p "+ "/csnsZone/home/public/raw/CSNS/TS1/BL02/"+runString+"/Monitor/Histogram"
+    subprocess.check_output(_tmp, shell=True)
+    _tmp = "imkdir -p "+ "/csnsZone/home/public/raw/CSNS/TS1/BL02/"+runString+"/Monitor/Event"
+    subprocess.check_output(_tmp, shell=True)
+    _tmp = "imkdir -p "+ "/csnsZone/home/public/nexus/CSNS/TS1/BL02/"+runString
+    subprocess.check_output(_tmp, shell=True)
+
+def addGroup():
+    BLname="BL02"
+    _tmp="iadmin mkuser "+BLname+"#csnsZone rodsuser"
+    subprocess.check_output(_tmp, shell=True)
+    _tmp="iadmin mkgroup "+"bl02admin"
+    subprocess.check_output(_tmp, shell=True)
+    _tmp="iadmin atg bl02admin "+BLname+"#csnsZone"
+    subprocess.check_output(_tmp, shell=True)
+
+    _tmp="iadmin mkgroup "+"bl02commissioning"
+    subprocess.check_output(_tmp, shell=True)
+    _tmp="iadmin atg bl02commissioning "+BLname+"#csnsZone"
+    subprocess.check_output(_tmp, shell=True)
+
+    _tmp="iadmin mkgroup "+"csnsuser"
+    subprocess.check_output(_tmp, shell=True)
+    _tmp="iadmin atg csnsuser "+BLname+"#csnsZone"
+    subprocess.check_output(_tmp, shell=True)
+
+def addAcl(runNo):
+    _tmp="ichmod -r read bl02commissioning /csnsZone/home/public/nexus/CSNS/TS1/BL02/"+getRunString(runNo)
+    subprocess.check_output(_tmp, shell=True)
+    _tmp="ichmod -r read bl02admin /csnsZone/home/public/raw/CSNS/TS1/BL02/"+getRunString(runNo)
+    subprocess.check_output(_tmp, shell=True)
+    _tmp="ichmod -r read bl02admin /csnsZone/home/public/nexus/CSNS/TS1/BL02/"+getRunString(runNo)
+    subprocess.check_output(_tmp, shell=True)
+    
+def putRods(sourcePath, targetPath):
+    retries=0
+    backupResc="csnsBackupDataResc"
+    restartFile="./rodsbase/restartFile"
+    lfRestartFile="./rodsbase/lfRestartFile"
+
+    putComOption="-KrTfP -X " + restartFile + " --retries "+ str(retries) + " --lfrestart " +lfRestartFile
+
+    _tmp = "iput -R " + backupResc +" "+putComOption +" "+ sourcePath +" "+ targetPath
+    
+    subprocess.call(_tmp, shell=True)
+
+    return True
+
+def putPublic(sourcePath, targetPath):
+    retries=0
+    backupResc="csnsResc"
+    restartFile="./rodsbase/restartFile"
+    lfRestartFile="./rodsbase/lfRestartFile"
+
+    putComOption="-KrTfP -X " + restartFile + " --retries "+ str(retries) + " --lfrestart " +lfRestartFile
+
+    _tmp = "iput -R " + backupResc +" "+putComOption +" "+ sourcePath +" "+ targetPath
+
+    subprocess.call(_tmp, shell=True)
+    
+    return True
+
+def transferNexusToPublic(runNo):
+    runString=getRunString(runNo)
+
+    mkdirPublic(runNo)
+
+    _b=time.time()
+
+    _nexusFiles      = getNexusFiles(runNo)
+
+    for _, _, _filesize, _fromlocation, _tolocation, _, _ in  _nexusFiles:
+        sourcePath=_fromlocation
+        targetPath="/csnsZone/home/public"+_tolocation
+        putPublic(sourcePath, targetPath)
+
+    addAcl(runNo)
+
+    print "Put Nexus To Public:", time.time()-_b
+    return True
+
+def transferNexusToBackup(runNo):
+    runString=getRunString(runNo)
+
+    mkdirRods(runNo)
+
+    _b=time.time()
+
+    _nexusFiles      = getNexusFiles(runNo)
+
+    for _, _, _filesize, _fromlocation, _tolocation, _, _ in  _nexusFiles:
+        sourcePath=_fromlocation
+        targetPath="/csnsZone/home/rods"+_tolocation
+        putRods(sourcePath, targetPath)
+
+    addAcl(runNo)
+
+    print "Put Nexus To Backup:", time.time()-_b
+    return True
+    
+def transferRawToPublic(runNo, conf):
+    runString=getRunString(runNo)
+
+    mkdirPublic(runNo)
+
+    _b=time.time()
+
+    _controlFiles=getControlFiles(runNo)
+    _detectorFiles=getDetectorFiles(runNo)
+    _monitorhisFiles = getMonitorhisFiles(runNo)
+    _monitorevtFiles = getMonitorevtFiles(runNo)
+
+    _nexusFiles      = getNexusFiles(runNo)
+    writeXml(runNo, conf['startDatetime'], conf['endDatetime'], _controlFiles, _detectorFiles, _monitorhisFiles,  _monitorevtFiles, _nexusFiles)
+    _softwareFiles       = getSoftwareFiles(runNo)
+
+    for _, _, _filesize, _fromlocation, _tolocation, _, _ in  _controlFiles:
+        sourcePath=_fromlocation
+        targetPath="/csnsZone/home/public"+_tolocation
+        putPublic(sourcePath, targetPath)
+
+    for _, _, _filesize, _fromlocation, _tolocation, _, _ in  _detectorFiles:
+        sourcePath=_fromlocation
+        targetPath="/csnsZone/home/public"+_tolocation
+        putPublic(sourcePath, targetPath)
+
+    for _, _, _filesize, _fromlocation, _tolocation, _, _ in  _monitorhisFiles:
+        sourcePath=_fromlocation
+        targetPath="/csnsZone/home/public"+_tolocation
+        putPublic(sourcePath, targetPath)
+
+    for _, _, _filesize, _fromlocation, _tolocation, _, _ in  _monitorevtFiles:
+        sourcePath=_fromlocation
+        targetPath="/csnsZone/home/public"+_tolocation
+        putPublic(sourcePath, targetPath)
+
+    addAcl(runNo)
+
+    print "Put Raw To Public:", time.time()-_b
+    return True
+
+def transferRawToBackup(runNo, conf):
+    runString=getRunString(runNo)
+
+    mkdirRods(runNo)
+
+    _b=time.time()
+    
+    _controlFiles=getControlFiles(runNo)
+    _detectorFiles=getDetectorFiles(runNo)
+    _monitorhisFiles = getMonitorhisFiles(runNo)
+    _monitorevtFiles = getMonitorevtFiles(runNo)
+
+    _nexusFiles      = getNexusFiles(runNo)
+    writeXml(runNo, conf['startDatetime'], conf['endDatetime'], _controlFiles, _detectorFiles, _monitorhisFiles,  _monitorevtFiles, _nexusFiles)
+    _softwareFiles       = getSoftwareFiles(runNo)
+
+    for _, _, _filesize, _fromlocation, _tolocation, _, _ in  _controlFiles:
+        sourcePath=_fromlocation
+        targetPath="/csnsZone/home/rods"+_tolocation
+        putRods(sourcePath, targetPath)
+
+    for _, _, _filesize, _fromlocation, _tolocation, _, _ in  _detectorFiles:
+        sourcePath=_fromlocation
+        targetPath="/csnsZone/home/rods"+_tolocation
+        putRods(sourcePath, targetPath)
+
+    for _, _, _filesize, _fromlocation, _tolocation, _, _ in  _monitorhisFiles:
+        sourcePath=_fromlocation
+        targetPath="/csnsZone/home/rods"+_tolocation
+        putRods(sourcePath, targetPath)
+
+    for _, _, _filesize, _fromlocation, _tolocation, _, _ in  _monitorevtFiles:
+        sourcePath=_fromlocation
+        targetPath="/csnsZone/home/rods"+_tolocation
+        putRods(sourcePath, targetPath)
+
+    for _, _, _filesize, _fromlocation, _tolocation, _, _ in  _softwareFiles:
+        sourcePath=_fromlocation
+        targetPath="/csnsZone/home/rods"+_tolocation
+        putRods(sourcePath, targetPath)
+
+    addAcl(runNo)
+
+    print "Put Raw To Backup:", time.time()-_b
+    return True
+
+def transferNexusToCloud(conf, runNo):
+    _b=time.time()
+
+    investigationName=conf['investigationName']
+
+    _path=getNexusPath(runNo)
+
+    _files=[]
+    try:
+        _file1=glob.glob(_path+"*.nxs")
+        _files=_file1
+    except Exception as ex:
+	print ex
+        pass
+
+    filelist=[]
+    for f in _files:
+        _filename=getFileName(f)
+        _fromlocation=f
+        _tolocation="/userData/BL02/" +investigationName+"/"+getRunString(runNo)+"/"+_filename
+
+        filelist.append([_fromlocation, _tolocation])
+
+    subprocess.check_output("mkdir -p /userData/BL02/"+investigationName+"/"+getRunString(runNo), shell=True)
+    for f in filelist:
+        _fromlocation, _tolocation=f
+        _cmd="/bin/cp "+_fromlocation+" "+_tolocation
+        _cmd=shlex.split(_cmd)
+        subprocess.call(_cmd, shell=False)
+
+    print "Put Nexus To Cloud:", time.time()-_b
+    return True
+
+# ==============================
+icatTime=time.time()
+sessionId, facilityId = getIcatInfo()
+# ==============================
+#try:
+#    addGroup()
+#except:
+#    pass
+
+class main(threading.Thread):
+    def __init__(self):
+        threading.Thread.__init__(self)
+	self.getRemoteDb()
+
+    def getRemoteDb(self):
+        engine = create_engine('mysql+mysqlconnector://transfer:Sanlie_123@mysql.csns.ihep.ac.cn/transfer_db')
+        #engine = create_engine('mysql+mysqlconnector://developer:sanlie_123@10.1.236.99/transfer_db')
+	self.DBSession = sessionmaker(bind=engine)
+
+    def openRemoteDb(self):
+        while True:
+            _result=False
+            try:
+                self.session = self.DBSession()
+                _result=True
+            except Exception as ex:
+                print ex
+                _result=False
+		time.sleep(5)
+                pass
+
+            if _result: break
+
+    def closeRemoteDb(self):
+        self.session.close()
+
+    def insertDb(self, user):
+        try:
+            self.session.add(user)
+            self.session.commit()
+            _success=True
+        except Exception as ex:
+	    print ex
+            _success=False
+
+        return _success
+
+    def queryDb(self, runNo):
+        runString=getRunString(runNo)
+        try:
+            _tmp = self.session.query(record).filter_by(run=runString).first()
+            _success1=True
+        except Exception as ex:
+	    print ex
+            _tmp=None
+            print "Mysql query failed"
+            _success1=False
+
+        if not _success1:
+            try:
+                self.openRemoteDb()
+                _tmp = self.session.query(record).filter_by(run=runString).first()
+                _success2=True
+            except Exception as ex:
+		print ex
+                _tmp=None
+                print "Mysql query failed"
+                _success2=False
+
+        _success = _success1 or _success2
+
+        if _tmp is not None:
+            _result=_tmp.stat
+        else:
+            _result=-1
+
+        return _success, _result
+
+    def updateDb(self, runNo, stat):
+        runString=getRunString(runNo)
+        try:
+            _tmp = self.session.query(record).filter_by(run=runString).first()
+            _tmp.stat = stat
+            self.session.add(_tmp)
+            self.session.commit()
+            _success=True
+        except Exception as ex:
+	    print ex
+            _success=False
+
+        return _success
+
+    def deleteDb(self, runNo):
+        pass
+
+    def createRecord(self, runNo, conf):
+        # ==============================
+        # has control summary file
+        hasControl = hasControlSummaryFile(runNo)
+        # ==============================
+        # has detector path
+        hasDetector = hasDetectorPath(runNo)
+        # ==============================
+        # has monitor histogram path
+        hasMonitorHis = hasMonitorHisPath(runNo)
+        # ==============================
+        # has monitor event path
+        hasMonitorEvt = hasMonitorEvtPath(runNo)
+        # ==============================
+        # has nexus path
+        hasNexus = hasNexusPath(runNo)
+
+        runString=getRunString(runNo)
+
+        user=record(beamline = 'BL02', run =runString, ctrl=hasControl, daq=hasDetector, monhis=hasMonitorHis, monevt=hasMonitorEvt, nexus=hasNexus, begin=conf['startDatetime'], end=conf['endDatetime'], stat=0)
+
+        return user
+
+    def getDb(self):
+        dbfile='bl02.db'
+        if os.path.exists(dbfile):
+            self.db = sqlite3.connect(dbfile,check_same_thread = False)
+            self.cur = self.db.cursor()
+        else:
+            self.db = sqlite3.connect(dbfile,check_same_thread = False)
+            self.cur = self.db.cursor()
+            tb = 'CREATE TABLE data (ID INTEGER PRIMARY KEY, BEAMLINE TEXT, RUN TEXT, CTRL INTEGER, DAQ INTEGER, MONHIS INTEGER, MONEVT INTEGER, NEXUS INTEGER,  BEGIN TEXT, END TEXT, STAT INTEGER)'
+            self.cur.execute(tb)
+            self.db.commit()
+
+    def setDb(self, runNo, conf):
+        # ==============================
+        # has control summary file
+        hasControl = hasControlSummaryFile(runNo)
+        # ==============================
+        # has detector path
+        hasDetector = hasDetectorPath(runNo)
+        # ==============================
+        # has monitor histogram path
+        hasMonitorHis = hasMonitorHisPath(runNo)
+        # has monitor event path
+        hasMonitorEvt = hasMonitorEvtPath(runNo)
+        # ==============================
+        # has nexus path
+        hasNexus = hasNexusPath(runNo)
+
+        runString=getRunString(runNo)
+        self.writeDb(runString, hasControl, hasDetector, hasMonitorHis, hasMonitorEvt, hasNexus,conf['startDatetime'], conf['endDatetime'], 0)
+
+    def deleteDb(self, STAT):
+        cur = self.db.cursor()
+        query = "delete from data where STAT = %d " % STAT
+        cur.execute(query)
+        self.db.commit()
+
+    def writeDb(self, RUN, CTRL, DAQ, MONHIS, MONEVT, NEXUS, BEGIN, END, STAT):
+        sqlstring = 'INSERT INTO data (ID, BEAMLINE, RUN, CTRL, DAQ, MONHIS, MONEVT, NEXUS,BEGIN, END, STAT) VALUES(NULL, "%s", "%s", "%d","%d", "%d", "%d", "%d", "%s", "%s", "%d")' %("BL02", RUN, CTRL, DAQ, MONHIS, MONEVT, NEXUS, BEGIN, END, STAT)
+        self.cur.execute(sqlstring)
+        self.db.commit()
+
+    def readDb(self):
+        self.cur.execute('SELECT * FROM data')
+        dbList = self.cur.fetchall()
+        #for i in dbList:
+        #    print i
+
+    def openDb(self):
+        dbfile='bl02.db'
+        self.db = sqlite3.connect(dbfile, check_same_thread = False)
+        self.cur = self.db.cursor()
+
+    def closeDb(self):
+        self.db.commit()
+        self.cur.close()
+        self.db.close()
+
+    def setDbStat(self, runNo, value):
+        runString=getRunString(runNo)
+        _tmp ="update data set STAT='%d' where run = '%s'" % (value, runString)
+        self.cur.execute(_tmp)
+
+    def fetchDbStat(self, id):
+        _tmp ="SELECT * FROM data where id = %d" % (id)
+        self.cur.execute(_tmp)
+        dbList = self.cur.fetchall()
+        for i in dbList:
+            print i
+
+    def getDbStat(self, runNo):
+        runString=getRunString(runNo)
+        self.cur.execute('SELECT * FROM data where run = ?', (runString,))
+        dbList = self.cur.fetchone()
+
+        return dbList[10]
+
+    def readControlSummaryFile(self, runNo):
+        _file=getControlSummaryFile(runNo)
+
+        if _file is not None:
+            with open(_file) as f:
+                xmlstring = f.read()
+
+            xmlstring=re.sub('\<NXroot.+', '<NXroot>', xmlstring)
+            root = ET.fromstring(xmlstring)
+
+            _tmp = root.findall("NXentry/start_time_utc")[0].text
+            if _tmp:
+                _tmp=re.sub(' ', 'T', _tmp)
+                _tmp+=".000+08:00"
+                startDatetime=_tmp
+            else:
+                startDatetime="unknown"
+
+            _tmp = root.findall('NXentry/end_time_utc')[0].text
+            if _tmp:
+                _tmp=re.sub(' ', 'T', _tmp)
+                _tmp+=".000+08:00"
+                endDatetime=_tmp
+            else:
+                endDatetime="unknown"
+
+            if startDatetime=="" and endDatetime!="":
+                startDatetime=endDatetime
+            elif startDatetime!="" and endDatetime=="":
+                endDatetime=startDatetime
+
+            _tmp1 = root.findall('NXentry/description')[0].text
+            _tmp2 = root.findall('NXentry/measurement_type')[0].text
+            if _tmp1 and _tmp2:
+                datasetDescription=_tmp1+":"+_tmp2
+            elif _tmp1:
+                datasetDescription=_tmp1
+            elif _tmp2:
+                datasetDescription=_tmp2
+            else:
+                datasetDescription="unknown"
+
+            user_id = root.findall('NXentry/user_id')[0].text
+            proposal_id = root.findall('NXentry/proposal_id')[0].text
+
+	    if (user_id is None) and (proposal_id is None):            
+    		user_id="BL02-user"
+                proposal_id="BL02-commissioning"
+
+            # create and get id from ICAT
+            _tmp = root.findall("NXentry/NXinstrument/NXsample/name")[0].text
+            if _tmp:
+                sampleTypeName=_tmp
+            else:
+                sampleTypeName="unknown"
+
+            sampleName=sampleTypeName
+
+            _tmp = root.findall("NXentry/NXinstrument/NXsample/chemical_formula")[0].text
+            if _tmp:
+                sampleTypeMolecularFormula=_tmp
+            else:
+                sampleTypeMolecularFormula="unknown"
+        else:
+    	    user_id="BL02-user"
+            proposal_id="BL02-commissioning"
+            startDatetime="unknown"
+            endDatetime="unknown"
+            datasetDescription="unknown"
+            sampleTypeName="unknown"
+            sampleName="unknown"
+            sampleTypeMolecularFormula="unknown"
+
+	# =================================
+        #user_id="BL02-user"
+        #proposal_id="BL02-commissioning"
+	# =================================
+
+        datasetName=getRunString(runNo)
+        dataLocation="/CSNS/TS1/BL02/"+datasetName
+	datasetName='__csns__'+datasetName
+
+        _dict={}
+        _dict['startDatetime']=startDatetime
+        _dict['endDatetime']=endDatetime
+        _dict['userName']=user_id
+        _dict['investigationName']=proposal_id
+        _dict['datasetName']=datasetName
+        _dict['dataLocation']=dataLocation
+        _dict['datasetDescription']=datasetDescription
+        _dict['sampleTypeName']=sampleTypeName
+        _dict['sampleName']=sampleName
+        _dict['sampleTypeMolecularFormula']=sampleTypeMolecularFormula
+
+        return _dict
+
+    # 0: begin
+    # 1: nexus transfer to Public
+    # 2: write icat
+    # 3: raw transfer to Public
+    # 4: write icat
+    # 5: nexus transfer to Backup
+    # 6: raw transfer to Backup
+    # 7: nexus transfer to Cloud
+    
+    def process(self):
+        global icatTime
+        global sessionId, facilityId
+
+        runNoList=getRunNoList(Nexuspath)
+
+	self.openRemoteDb()
+
+        for runNo in runNoList:
+            access, stat = self.queryDb(runNo)
+            #if not access: continue
+
+            if getSessionInfo(sessionId) is None:
+                print "Try login into ICAT"
+                sessionId = getSessionId()
+
+            if stat!=6:
+                print "begin: ", runNo, stat
+                conf = self.readControlSummaryFile(runNo)
+
+                investigationName=conf['investigationName']
+                visitName=investigationName
+                try:
+                    investigationId=getInvestigationId(sessionId, investigationName, visitName)
+                except Exception as ex:
+                    print ex
+                    try:
+                        i_temp = "P" + investigationName
+                        v_temp = i_temp
+                        print "Try", i_temp
+                        investigationId = getInvestigationId(sessionId, i_temp, v_temp)
+                        investigationName = i_temp
+                        visitName = v_temp
+                        conf['investigationName'] = investigationName
+                    except Exception as ex:
+                        print ex
+                        try:
+                            i_temp = "BL02-commissioning"
+                            v_temp = i_temp
+                            print "Try", i_temp
+                            investigationId=getInvestigationId(sessionId, i_temp, v_temp)
+                            investigationName = i_temp
+                            visitName = v_temp
+                            conf['investigationName'] = investigationName
+                        except Exception as ex:
+                            print "investigation not found"
+                            continue
+
+                if not investigationId: continue
+                user = self.createRecord(runNo, conf)
+
+            if stat==-1:
+                access=self.insertDb(user)
+                if access:
+                    stat=0
+                else:
+                    continue
+
+            if stat==0:
+                success1 = transferNexusToPublic(runNo)
+                success2 = transferNexusToCloud(conf, runNo)
+                #success2=True
+		if success1 and success2:
+                    access=self.updateDb(runNo, 1)
+                    if access:
+                        stat=1
+                    else:
+                        continue
+
+            if stat==1:
+                success = writeNexusToIcat(runNo, conf, investigationId)
+                if success:
+                    access=self.updateDb(runNo, 2)
+                    if access:
+                        stat=2
+                    else:
+                        continue
+
+            if stat==2:
+                success = transferRawToPublic(runNo, conf)
+                if success:
+                    access=self.updateDb(runNo, 3)
+                    if access:
+                        stat=3
+                    else:
+                        continue
+
+            if stat==3:
+                success = writeRawToIcat(runNo, conf, investigationId)
+                setInvestigationDate(conf, investigationId)
+
+                if success:
+                    access=self.updateDb(runNo, 4)
+                    if access:
+                        stat=4
+                    else:
+                        continue
+
+            if stat==4:
+                success = transferNexusToBackup(runNo)
+                if success:
+                    access=self.updateDb(runNo, 5)
+                    if access:
+                        stat=5
+                    else:
+                        continue
+
+            if stat==5:
+                success = transferRawToBackup(runNo, conf)
+                if success:
+                    access=self.updateDb(runNo, 6)
+                    if access:
+                        stat=6
+                        print "end: ", runNo
+                    else:
+                        continue
+
+            if stat==6:
+                pass
+
+        self.closeRemoteDb()
+
+    def run(self):
+        while True:
+            self.process()
+            time.sleep(60.0)
+
+if __name__=="__main__":
+
+    threadDaemon = main()
+    threadDaemon.setDaemon(True)
+    threadDaemon.start()
+    threadDaemon.join()
+    sys.exit()
+
