text: #-------------------------------------------------------------------------------
  # Copyright 2016 Congduc Pham, University of Pau, France.
  # 
  # Congduc.Pham@univ-pau.fr
  #
  # This file is part of the low-cost LoRa gateway developped at University of Pau
  #
  # This program is free software: you can redistribute it and/or modify
  # it under the terms of the GNU General Public License as published by
  # the Free Software Foundation, either version 3 of the License, or
  # (at your option) any later version.
  #
  # This program is distributed in the hope that it will be useful,
  # but WITHOUT ANY WARRANTY; without even the implied warranty of
  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  # GNU General Public License for more details.
  # 
  # You should have received a copy of the GNU General Public License
  # along with the program.  If not, see <http://www.gnu.org/licenses/>.
  #-------------------------------------------------------------------------------
  # Jul/2016 adapted by N. Bertuol under C. Pham supervision 
  # 
  # nicolas.bertuol@etud.univ-pau.fr
  #
  # Oct/2016. re-designed by C. Pham to use self-sufficient script per cloud
  # Feb/2018. add key association, field association and nomenclature association features
  #
  # general format of data is now thingspeak_channel#thingspeak_field#TC/22.5/HU/24/LU/345/CO2/456... 
  # ex: ##TC/22.5/HU/24... or TC/22.5/HU/24... or thingspeak_channel##TC/22.5/HU/24... or thingspeak_channel#TC/22.5/HU/24... or 
  #	  #thingspeak_field#TC/22.5/HU/24... or thingspeak_field#TC/22.5/HU/24... 
  # where some default value can be used
  #
  # key_ThingSpeak.py defines: 
  #	- _def_thingspeak_channel_key
  #	- source_list
  #	- key_association
  #	- field_association
  #	- nomenclature_association
  # for instance:
  #	- _def_thingspeak_channel_key='SGSH52UGPVAUYG3S'
  #	- source_list=["6", "7", "8", "9"]
  #	- key_association=[('AAAAAAAAAAAAAAAA', 9), ('BBBBBBBBBBBBBBBB', 10, 11)]
  #		- node 9 will use channel AAAAAAAAAAAAAAAA
  #		- node 10 and 11 will use channel BBBBBBBBBBBBBBBB
  # 		- other nodes will use default channel
  #		- note that priority is given to key association defined on gateway
  #		- with key association you can go beyond the limitation of 8 charts per channel: if you have many sensors, you can assign specific channel's write key to specific sensors
  #	- field_association=[(6,1),(7,5)]
  #		- [(6,1),(7,5)] means data from respectively sensor 6/7 will use starting field index of 1/5 
  #	- nomenclature_association=[("TC",0),("HU",1),("LU",2),("CO2",3)]
  #		- ("TC",0) means that if nomemclature is "TC" then the offset for field index will be 0
  #		
  # Examples with the above definitions, each sensor node has 4 physical sensors: TC, HU, LU, CO2 
  #	- if we receive "TC/22.5/HU/24/LU/345/CO2/456" from sensor 6
  #		- data will be accepted
  #		- starting field index for the channel will be 1
  #		- TC will be uploaded on field 1, HU on field 2, LU on field 3 and CO2 on field 4
  #	- if we receive "HU/24/LU/345" from sensor 6 because these physical measures are sent at higher frequency
  #		- HU and LU will still be respectively uploaded on field 2 and 3
  #	- if we receive "TC/22.5/HU/24/LU/345/CO2/456" from sensor 7
  #		- data will be accepted
  #		- starting field index for the channel will be 5
  #		- TC will be uploaded on field 5, HU on field 6, LU on field 7 and CO2 on field 8	
  #		- in this example, with 4 physical sensors per node, then a ThingSpeak channel can handle 2 nodes
  #	- if we receive "CCCCCCCCCCCCCCCC#TC/22.5/HU/24/LU/345/CO2/456" from sensor 8
  #		- data will be accepted
  #		- data will be uploaded on the channel which write key is "CCCCCCCCCCCCCCCC" as there is no key association defined
  #		- starting field index will be the default value, i.e. 1
  #		- TC will be uploaded on field 1, HU on field 2, LU on field 3 and CO2 on field 4
  #	- if we receive "CCCCCCCCCCCCCCCC#5#TC/22.5/HU/24/LU/345/CO2/456" from sensor 8
  #		- data will be accepted
  #		- data will be uploaded on the channel which write key is "CCCCCCCCCCCCCCCC"
  #		- starting field index will be 5
  #		- TC will be uploaded on field 5, HU on field 6, LU on field 7 and CO2 on field 8
  #	- if we receive "CCCCCCCCCCCCCCCC#TC/22.5/HU/24/LU/345/CO2/456" from sensor 9
  #		- data will be accepted
  #		- data will be uploaded on the channel which write key is "AAAAAAAAAAAAAAAA" as there **is** a key association that has priority
  #		- starting field index will be the default value, i.e. 1
  #		- TC will be uploaded on field 1, HU on field 2, LU on field 3 and CO2 on field 4
  #	- if we receive "TC/22.5/HU/24/AA/345/BB/456" from sensor 6
  #		- data will be accepted
  #		- starting field index for the channel will be 1
  #		- TC will be uploaded on field 1, HU on field 2, AA on field 3 and BB on field 4 
  #		  even if AA and BB have no nomenclature association
  #	- if we receive "YY/30/TC/22.5/AA/345/BB/456" from sensor 6
  #		- data will be accepted
  #		- starting field index for the channel will be 1
  #		- YY will be uploaded on field 1, TC on field 1, AA on field 3 and BB on field 4
  #		  here YY has no nomenclature association and you can see that TC overwrite YY on field 1
  #	- if we receive "TC/22.5/HU/24/LU/345/CO2/456" from sensor 10
  #		- data will be discarded by CloudThingSpeak.py
  #
  # Congduc.Pham@univ-pau.fr
  
  import urllib2
  import subprocess
  import time
  import ssl
  import socket
  import datetime
  import sys
  import re
  
  #don't generate pyc (no compilation of imported module) so change in key_* file can be done dynamically
  sys.dont_write_bytecode = True
  
  # get key definition from external file to ease
  # update of cloud script in the future
  import key_ThingSpeak
  
  try:
  	key_ThingSpeak.source_list
  except AttributeError:
  	key_ThingSpeak.source_list=[]
  
  try:
  	key_ThingSpeak.field_association
  except AttributeError:
  	key_ThingSpeak.field_association=[]
  
  try:
  	key_ThingSpeak.nomenclature_association
  except AttributeError:
  	key_ThingSpeak.nomenclature_association=[]
  
  try:
  	key_ThingSpeak.key_association
  except AttributeError:
  	key_ThingSpeak.key_association=[]
  			
  # didn't get a response from thingspeak server?
  connection_failure = False
  
  # retry if return from server is 0?
  retry = False
  
  # function to check connection availability with the server
  def test_network_available():
  	connection = False
  	iteration = 0
  	response = None
  	
  	# we try 4 times to connect to the server.
  	while(not connection and iteration < 4):
  		try:
  			# 3sec timeout in case of server available but overcrowded
  			response=urllib2.urlopen('https://api.thingspeak.com/', timeout=3)
  			connection = True
  		except urllib2.URLError, e: pass
  		except socket.timeout: pass
  		except ssl.SSLError: pass
  	    	
  		# if connection_failure == True and the connection with the server is unavailable, don't waste more time, exit directly
  		if (connection_failure and response is None):
  			print('Thingspeak: the server is still unavailable')
  			iteration = 4
  	    # print connection failure
  		elif (response is None):
  			print('Thingspeak: server unavailable, retrying to connect soon...')
  			# wait before retrying
  			time.sleep(1)
  			iteration += 1
  
  	return connection
  	
  #upload multiple data
  #this is the only way to upload data 
  #format of data is now TC/22.5/HU/24/LUM/345...
  
  def thingspeak_uploadMultipleData(data, src, nomenclatures):
  	global connection_failure
  	
  	connected = test_network_available()
  	
  	# if we got a response from the server, send the data to it
  	if(connected):
  		connection_failure = False
  		
  		print("ThingSpeak: uploading (multiple)")
  		print 'rcv msg to log (\!) on ThingSpeak (',			
  
  		found_key_association=False
  
  		# check if we have a key_association field
  		# priority is given to key association defined on gateway
  		if (len(key_ThingSpeak.key_association)!=0):
  			for k in range(0, len(key_ThingSpeak.key_association)):
  				# found a key for this source node
  				if src in key_ThingSpeak.key_association[k][1:]:
  					found_key_association=True
  					data[0]=key_ThingSpeak.key_association[k][0]
  					print data[0],
  
  		if (len(key_ThingSpeak.key_association)==0 or found_key_association==False):
  			#use default channel?
  			if data[0]=='':
  				data[0]=key_ThingSpeak._def_thingspeak_channel_key
  				print 'default',
  			else:	
  				print data[0],
  			
  		print ',',
  
  		if (len(key_ThingSpeak.field_association)==0):
  			defined_field_association=False
  		else:
  			defined_field_association=True
  
  		if (len(key_ThingSpeak.nomenclature_association)==0):
  			defined_nomenclature_association=False
  		else:
  			defined_nomenclature_association=True
  			
  		found_field_association=False
  			
  		#do we have key_ThingSpeak.field_association defined?
  		#again, priority is given to field association defined on gateway
  		if defined_field_association==True:
  			i = 0
  			while i < len(key_ThingSpeak.field_association):
  				if key_ThingSpeak.field_association[i][0]==src:
  					#found a field for that sensor
  					found_field_association=True
  					fieldNumber=key_ThingSpeak.field_association[i][1]
  					print fieldNumber,
  				i = i+1
  					
  		if (defined_field_association==False) or (found_field_association==False):
  			#use default field?
  			if data[1]=='':
  				fieldNumber = 1
  				print 'default',
  			else:
  				fieldNumber = int(data[1])
  				print data[1],				
  	
  		print '): '
  		
  		if found_field_association==True:
  			print 'field index set by field association for sensor '+str(src)
  		
  		i=0
  
  		cmd = 'curl -s -k -X POST --data '
  								
  		while i < len(data)-2:
  					
  			#use nomemclature association
  			field_offset=0
  			the_fieldNumber=fieldNumber+i
  			
  			if (defined_nomenclature_association):
  				for item in key_ThingSpeak.nomenclature_association:
  					if nomenclatures[i]==item[0]:
  						print 'found a nomenclature association for '+nomenclatures[i]
  						field_offset=item[1]
  						print 'will use field offet of '+str(field_offset)
  						the_fieldNumber=fieldNumber+field_offset
  						print 'field index will be '+str(the_fieldNumber)
  					
  			if (i==0):
  				cmd += 'field'+str(the_fieldNumber)+'='+data[i+2]
  			else:
  				cmd += '&field'+str(the_fieldNumber)+'='+data[i+2]
  				
  			i += 1
  			
  		cmd += ' https://api.thingspeak.com/update?key='+data[0]
  		
  		print("ThingSpeak: will issue curl cmd")
  		print(cmd)
  		args = cmd.split()
  	
  		#retry enabled
  		if (retry) :
  			out = '0'
  			iteration = 0
  		
  			while(out == '0' and iteration < 6 and not connection_failure) :
  				try:
  					out = subprocess.check_output(args, shell=False)
  
  					#if server return 0, we didn't wait 15sec, wait then
  					if(out == '0'):
  						print('ThingSpeak: less than 15sec between posts, retrying in 3sec')
  						iteration += 1
  						time.sleep( 3 )
  					else:
  						print('ThingSpeak: returned code from server is %s' % out)
  				except subprocess.CalledProcessError:
  					print("ThingSpeak: curl command failed (maybe a disconnection)")
  					
  					#update connection_failure
  					connection_failure = True
  				
  		#retry disabled
  		else :
  			try:
  				out = subprocess.check_output(args, shell=False)
  				if (out == '0'):
  					print('ThingSpeak: returned code from server is %s, do not retry' % out)
  				else :
  					print('ThingSpeak: returned code from server is %s' % out)
  				
  			except subprocess.CalledProcessError:
  				print("ThingSpeak: curl command failed (maybe a disconnection)")
  				connection_failure = True
  	else:
  		connection_failure = True
  		
  	if(connection_failure):
  		print("ThingSpeak: not uploading")
  			
  	
  def thingspeak_setRetry(retry_bool):
  
  	global retry
  	retry = retry_bool
  	
  # main
  # -------------------
  
  def main(ldata, pdata, rdata, tdata, gwid):
  
  	#this is common code to process packet information provided
  	#by the main gateway script (i.e. post_processing_gw.py)
  	#these information are provided in case you need them	
  	arr = map(int,pdata.split(','))
  	dst=arr[0]
  	ptype=arr[1]				
  	src=arr[2]
  	seq=arr[3]
  	datalen=arr[4]
  	SNR=arr[5]
  	RSSI=arr[6]
  	
  	#LoRaWAN packet
  	if dst==256:
  		src_str="0x%0.8X" % src
  	else:
  		src_str=str(src)	
  
  	if (src_str in key_ThingSpeak.source_list) or (len(key_ThingSpeak.source_list)==0):
  
  		#remove any space in the message as we use '/' as the delimiter
  		#any space characters may introduce error
  		ldata=ldata.replace(' ', '')
  				
  		#this part depends on the syntax used by the end-device
  		#we use: thingspeak_channel#thingspeak_field#TC/22.4/HU/85... 
  		#ex: ##TC/22.4/HU/85... or TC/22.4/HU/85... or thingspeak_channel##TC/22.4/HU/85... 
  		#or #thingspeak_field#TC/22.4/HU/85... to use some default value
  				
  		# get number of '#' separator
  		nsharp = ldata.count('#')	
  				
  		#no separator
  		if nsharp==0:
  			#will use default channel and field
  			data=['','']
  		
  			#contains ['', '', "s1", s1value, "s2", s2value, ...]
  			data_array = data + re.split("/", ldata)		
  		elif nsharp==1:
  			#only 1 separator
  		
  			data_array = re.split("#|/", ldata)
  		
  			#if the first item has length > 1 then we assume that it is a channel write key
  			if len(data_array[0])>1:
  				#insert '' to indicate default field
  				data_array.insert(1,'');		
  			else:
  				#insert '' to indicate default channel
  				data_array.insert(0,'');		
  		else:
  			#contains [channel, field, "s1", s1value, "s2", s2value, ...]
  			data_array = re.split("#|/", ldata)	
  		
  		#just in case we have an ending CR or 0
  		data_array[len(data_array)-1] = data_array[len(data_array)-1].replace('\n', '')
  		data_array[len(data_array)-1] = data_array[len(data_array)-1].replace('\0', '')	
  	
  		#test if there are characters at the end of each value, then delete these characters
  		i = 3
  		while i < len(data_array) :
  			while not data_array[i][len(data_array[i])-1].isdigit() :
  				data_array[i] = data_array[i][:-1]
  			i += 2
  		
  		#get number of '/' separator
  		nslash = ldata.count('/')
  	
  		nomenclatures = []	
  		#data to send to thingspeak
  		data = []
  		data.append(data_array[0]) #channel (if '' default)
  		data.append(data_array[1]) #field (if '' default)		
  
  		if nslash==0:
  			# old syntax without nomenclature key, so insert only one key
  			# we use DEF
  			nomenclatures.append("DEF")
  			data.append(data_array[2])
  		else:
  			# completing nomenclatures and data
  			i=2
  			while i < len(data_array)-1 :
  				nomenclatures.append(data_array[i])
  				data.append(data_array[i+1])
  				i += 2
  					
  		thingspeak_uploadMultipleData(data, src, nomenclatures)
  	else:
  		print "Source is not is source list, not sending with CloudThingSpeak.py"				
  	
  if __name__ == "__main__":
  	main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5])
  	
  
  