Issue #11 changes

client/iocBoot/iocdemo/st_test.cmd

@@ -0,0 +1,23 @@
+#!../../bin/linux-x86_64-debug/demo
+
+## You may have to change demo to something else
+## everywhere it appears in this file
+
+< envPaths
+
+## Register all support components
+dbLoadDatabase("../../dbd/demo.dbd",0,0)
+demo_registerRecordDeviceDriver(pdbbase) 
+
+var(reccastTimeout, 5.0)
+var(reccastMaxHoldoff, 5.0)
+
+epicsEnvSet("IOCNAME", "myioc")
+epicsEnvSet("ENGINEER", "myself")
+epicsEnvSet("LOCATION", "myplace")
+
+
+## Load record instances
+dbLoadRecords("../../db/cfstore_test_records.db", "P=test:")
+
+iocInit()

server/demo.conf

@@ -56,3 +56,10 @@ dbname = test.db
 # Must be unique for each instance accessing
 # a common database.
 idkey = 42
+
+[cf]
+# cf-store application
+
+# Debug output file location.
+# Produces no file when not defined.
+# debug_file_loc = /home/devuser/recsyncdata.txt

server/recceiver/interfaces.py

@@ -9,7 +9,7 @@ class ITransaction(Interface):
     src = Attribute('Source Address.')
 
     addrec = Attribute("""Records being added
-    recid: ('recname', 'rectype', {'key':'val'})
+    {recid: ('recname', 'rectype', {'key':'val'})}
     """)
 
     delrec = Attribute('A set() of recids which are being removed')

server/recceiver/mock_client.py

@@ -0,0 +1,89 @@
+from twisted.internet.address import IPv4Address
+from requests import HTTPError
+class mock_client():
+    def __init__(self):
+        self.cf = {}
+        self.connected = True
+        self.fail_find = False
+        self.fail_set = False
+
+    def findByArgs(self, args):
+        if not self.connected:
+            raise HTTPError("Mock Channelfinder Client HTTPError", response=self)
+        else:
+            result = []
+
+            if len(args) > 1:  # returning old
+                for ch in self.cf:
+                    name_flag = False
+                    prop_flag = False
+                    for props in self.cf[ch][u'properties']:
+                        if props[u'name'] == args[0][0]:
+                            if props[u'value'] == args[0][1]:
+                                name_flag = True
+                        if props[u'name'] == args[1][0]:
+                            if props[u'value'] == args[1][1]:
+                                prop_flag = True
+                    if name_flag and prop_flag:
+                        result.append(self.cf[ch])
+                return result
+            else:
+                if args[0][0] == '~name' and args[0][1] in self.cf:
+                    return [self.cf[args[0][1]]]
+                if args[0][0] == 'pvStatus' and args[0][1] == 'Active':
+                    for ch in self.cf:
+                        for prop in self.cf[ch]['properties']:
+                            if prop['name'] == 'pvStatus':
+                                if prop['value'] == 'Active':
+                                    result.append(self.cf[ch])
+                    return result
+
+    def findProperty(self, prop_name):
+        if not self.connected:
+            raise HTTPError("Mock Channelfinder Client HTTPError", response=self)
+        else:
+            if prop_name in ['hostName', 'iocName', 'pvStatus', 'time']:
+                return prop_name
+
+    def set(self, channels):
+        if not self.connected or self.fail_set:
+            raise HTTPError("Mock Channelfinder Client HTTPError", response=self)
+        else:
+            for channel in channels:
+                self.addChannel(channel)
+
+    def update(self, property, channelNames):
+        if not self.connected or self.fail_find:
+            raise HTTPError("Mock Channelfinder Client HTTPError", response=self)
+        else:
+            for channel in channelNames:
+                self.__updateChannelWithProp(property, channel)
+
+    def addChannel(self, channel):
+        self.cf[channel[u'name']] = channel
+
+    def __updateChannelWithProp(self, property, channel):
+        if channel in self.cf:
+            for prop in self.cf[channel]['properties']:
+                if prop['name'] == property['name']:
+                    prop['value'] = property['value']
+                    prop['owner'] = property['owner']  # also overwriting owner because that's what CF does
+                    return
+
+class mock_conf():
+    def __init__(self):
+        pass
+
+    def get(self, name, target):
+        return "cf-update"
+
+class mock_TR():
+    def __init__(self):
+        self.addrec = {}
+        self.src = IPv4Address('TCP', 'testhosta', 1111)
+        self.delrec = ()
+        self.infos = {'CF_USERNAME': 'cf-update', 'ENGINEER': 'cf-engi'}
+        self.initial = True
+        self.connected = True
+        self.fail_set = False
+        self.fail_find = False

server/recceiver/recast.py

@@ -87,6 +87,7 @@ def timed(self):
             self.phase = 2
             self.nonce = random.randint(0,0xffffffff)
             self.writeMsg(0x8002, _ping.pack(self.nonce))
+            _log.debug("ping nonce: " + str(self.nonce))
 
     def getInitialState(self):
         return (self.recvHeader, 8)
@@ -124,6 +125,7 @@ def recvPong(self, body):
             _log.error('pong nonce does not match! %s!=%s',nonce,self.nonce)
             self.transport.loseConnection()
         else:
+            _log.debug('pong nonce match')
             self.phase = 1
         return self.getInitialState()
 
@@ -241,7 +243,7 @@ def flush(self, connected=True):
         op = self.factory.commit(TR)
         if op:
             op.addCallbacks(self.resume, self.abort)
-            self.proto.transport.pauseProducing()
+            #self.proto.transport.pauseProducing()
             self.op = op
 
     def resume(self, arg):

server/recceiver/scripts/add_extra_properties.py

@@ -0,0 +1,23 @@
+from channelfinder import ChannelFinderClient
+
+'''
+Simple script for adding active channels to Channel Finder Service for testing cf-store clean
+If it gives a 500 error, run it again. Glassfish and CFS must be set up and running.
+'''
+
+
+def abbr(name, hostname, iocname, status):
+    return {u'owner': 'cf-update',
+            u'name': name,
+            u'properties': [
+                {u'owner': 'cf-update', u'name': 'hostName',
+                 u'value': hostname},
+                {u'owner': 'cf-update', u'name': 'iocName',
+                 u'value': iocname},
+                {u'owner': 'cf-update', u'name': 'pvStatus',
+                 u'value': status},
+                {u'owner': 'cf-update', u'name': 'time',
+                 u'value': '2016-08-18 12:33:09.953985'}]}
+
+client = ChannelFinderClient()
+client.set(channels=[abbr(u'ch1', 'testhosta', 1111, 'Active'), abbr(u'test_channel', 'testhosta', 1111, 'Active')])

server/recceiver/scripts/print_cf_data.py

@@ -0,0 +1,36 @@
+from channelfinder import ChannelFinderClient
+import json
+import os
+from operator import itemgetter
+filename = "/home/devuser/cfdata"  # change this to output file name
+client = ChannelFinderClient()
+
+
+def get_cf_data(client):
+    channels = client.findByArgs([('pvStatus', 'Active')])
+
+    for ch in channels:
+        ch.pop('owner', None)
+        ch.pop('tags', None)
+        for prop in ch['properties']:
+            if prop['name'] == 'hostName':
+                ch['hostName'] = prop['value']
+            if prop['name'] == 'iocName':
+                ch['iocName'] = prop['value']
+        ch.pop('properties', None)
+    return channels
+
+channels = get_cf_data(client)
+
+if os.path.isfile(filename):
+    os.remove(filename)
+
+new_list = []
+
+for channel in channels:
+    new_list.append([channel['name'], channel['hostName'], int(channel['iocName'])])
+
+new_list.sort(key=itemgetter(0))
+
+with open(filename, 'wrx') as f:
+    json.dump(new_list, f)

server/recceiver/scripts/readme

@@ -0,0 +1,75 @@
+MANUAL TEST LOG:
+
+PASSED - Clean and poll on recsync start
+PASSED - Clean and poll on recsync close
+PASSED - Clean and poll on IOC start
+PASSED - Clean and poll on IOC close
+PASSED - 100 IOCs overlapping 1 IOC
+
+Clean and poll on recsync start:
+0.a CF on
+0.b add_extra_properties
+    - should see the new props in CF
+1. CF off
+2. Start recsync
+    - will poll in clean
+3. Enable CF
+    - polling completes
+4. CF will clean
+5. Check result
+    - all channels inactive
+
+Clean and poll on recsync close:
+1. CF on
+2. Start recsync
+3. add_extra_properties
+    - should see new props in CF
+4. CF off
+5. Recsync off
+    - will poll in clean
+    - (NYI) will give up if waiting too long
+6. CF on
+7. Check result
+    - all channels inactive
+
+Commit poll on IOC start:
+1. Enable CF
+2. Start recsync
+    - will perform clean
+3. Disable CF
+4. Start IOC
+    - waits for recsync announcement
+    - connects and polls commit
+5. Enable CF
+6. Commit poll fails once, reconnects, and commits
+7. Check result
+    - all channels active
+
+Commit poll on IOC close:
+1. Enable CF
+2. Start Recsync
+3. Start IOC
+4. Wait for IOC to connect and commit
+    - all IOC channels now active
+5. Disable CF
+6. Close IOC
+    - commits [] and polls
+7. Enable CF
+    - reconnects after polling and completes commit
+8. Check result
+    - all channels inactive
+
+100 IOCs overlapping 1 IOC:
+0. * Disable logging in demo.conf *
+1. Enable CF
+2. Start recsync
+3. Start st.cmd in a command prompt
+    - IOC will connect and commit channels
+    - 4 channels will be active
+4. run test_mock_iocs.py
+    - wait forever
+    - all channels will be active
+    - 2 channels will still belong to st.cmd
+5. kill test_mock_iocs.py or killall st_test.cmd
+    - wait forever again
+    - 4 channels will be active, all belonging to st.cmd

server/recceiver/scripts/test_mock_iocs.py

@@ -0,0 +1,52 @@
+import os
+import threading
+import sys
+import time
+import signal
+
+iocexecutable = "st.cmd"
+
+
+def startIOC():
+    # conf needs to be set
+    pid, fd = os.forkpty()
+    if pid == 0:
+        os.chdir("../../../client/iocBoot/iocdemo")
+        print os.curdir
+        os.execv("st_test.cmd", [''])
+    return pid, fd
+
+
+def readfd(fd):
+    while 1:
+        empt = str(os.read(fd, 16384).strip("\n"))
+
+
+def handler(signum, frame):
+    global pids
+    for pid in pids:
+        os.kill(pid, signal.SIGKILL)
+    sys.exit()
+
+
+def main():
+    global pids
+    pids = []
+    signal.signal(signal.SIGTERM, handler)
+    os.chdir(os.path.dirname(os.path.abspath(sys.argv[0])))  # Uses a filename, not good, also only works on linux?
+    threads = []
+    for i in range(1, 100):
+        iocpid, iocfd = startIOC()
+        pids.append(iocpid)
+        print "len pids: ", len(pids)
+        iocthread = threading.Thread(group=None, target=readfd, args=(iocfd,), name="iocthread", kwargs={})
+        threads.append(iocthread)
+        iocthread.start()
+    try:
+        while 1:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        sys.exit()
+
+if __name__ == '__main__':
+    main()