import os
from allegroUtils.wvr.wvr import *
from allegroUtils.wvr.wvrtools import *
#%from importfunct import *
from allegroUtils.casacore import *
from allegroUtils.core import *
###########################################
# MySQL Tools
##########################################
[docs]def checkentry(results):
''' Checks if the MySQL database already contains information on the UID in questions
:param results: UID name
:type resulsts: str
:returns: None or string with missing entries
:rtype: str
'''
try:
uid,data_type,suffix=results.split('.')
d = fits.open(results)
# get the antenna info of the baselines
D = d[2].data
baselinenames = D['name'][:]
# mySQL check bit
# con = lite.connect('/lustre/allegro/allegro_staff/projects/allegro/wvr/masterdatabase/compare1.db')
con = lite.connect('/lustre/allegro/allegro_staff/projects/allegro/wvr/masterdatabase/comparenew.db')
cur = con.cursor() # cursor
missing = False
for bl in baselinenames:
cur.execute("SELECT * FROM Overview WHERE uid = ? AND blname = ?",(uid,bl))
data=cur.fetchone()
if data is None:
missing = True
#print ('Complete Entry needed for %s, baseline %s' % (uid,bl))
elif any(x is None for x in data):
missing = True
#print ('Check Entry. Incomplete info for %s, baseline %s' % (uid,bl))
if missing is False :
print ('******* MS already ingested correctly **********')
else:
print ('******* Entries missing or wrong in database **********')
con.close()
return missing
except:
print "Error in checkentry"
[docs]def putentry(results):
''' OUT OF DATE!!!
Enters the required values of a reduced asdm into the old style sql databas (compare2.db)
:param results: UID name of the fits file
:type resulsts: str
:returns: None
:rtype: None
'''
try:
uid,data_type,suffix=results.split('.')
data = fits.open(results)
#get the header info
PWV = data[0].header['PWV']
PWVERR = data[0].header['PWVERR']
RestFreq = data[0].header['RESTFREQ']
elevation = data[0].header['ALT']
azimuth = data[0].header['AZ']
source = data[0].header['SOURCE']
fluxdens = data[0].header['FLUXDENS']
D = data[2].data
baselinenames = D['name'][:]
lengths = D['xy'][:]
A = data[1].data
bl_dirs = get_bldir(A,baselinenames)
W = data[3].data
wind_speed = np.mean(W['wind_speed']) ; temperature = np.mean(W['temperature'])
wind_dir = np.mean(W['wind_direction']); rel_hum = np.mean(W['rel_humidity'])
pressure = np.mean(W['pressure'])
comment = "No comment" # not used currently
M = data[8].data
# mySQL check bit
# con = lite.connect('/lustre/allegro/allegro_staff/projects/allegro/wvr/masterdatabase/compare1.db') # old database, no elevation, az, source name or fluxdens
con = lite.connect('/lustre/allegro/allegro_staff/projects/allegro/wvr/masterdatabase/comparenew.db')
cur = con.cursor() # cursor
for idx,base in enumerate(baselinenames):
length=np.float(lengths[idx]);bl_dir=np.float(bl_dirs[idx])
for i,j in enumerate(M['name']):
if j == base:
rms_phase = np.float(np.degrees(M['phase_rms'][i]));rms_corr=np.float(np.degrees(M['corr_rms'][i]));improv=np.float(M['phase_rms'][i]/M['corr_rms'][i])
q = """
INSERT INTO Overview(uid, blname, windspeed, temperature, winddir, rel_humidity, pressure, pwv, source, elevation, azimuth, fluxdens, restfreq, bl_length, bl_dir, rms_phase, rms_corr, ratio, comment)
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'No Comment')
"""
cur.execute(q,(uid,base,wind_speed,temperature,wind_dir, rel_hum, pressure, PWV,source, elevation, azimuth, fluxdens, RestFreq,length,bl_dir,rms_phase,rms_corr,improv))
con.commit()
except:
print "ERROR in putentry", sys.exc_info()[0]
raise
[docs]def putentry2(results):
''' Out of Date, use putentry3
Enters the required values of a reduced asdm into the new style sql databas (comparenew.db)
:param results: UID fits file
:type resulsts: str
:returns: None
:rtype: None
'''
try:
uid,data_type,suffix=results.split('.')
data = fits.open(results)
W = data[3].data
con = lite.connect('/lustre/allegro/allegro_staff/projects/allegro/wvr/masterdatabase/comparenew.db')
cur = con.cursor()
#get the header info
timescale = [1.1,12.,32.,64.,128.]
# ARRAY PARAMETERS
RestFreq = float(data[0].header['RESTFREQ'])
elevation = float(data[0].header['ALT'])
azimuth = float(data[0].header['AZ'])
source = data[0].header['SOURCE']
fluxdens = float(data[0].header['FLUXDENS'])
wind_speed_mean = float(np.mean(W['wind_speed']))
wind_speed_rms = float(np.sqrt(np.mean((W['wind_speed']-wind_speed_mean)**2)))
temperature_mean = float(np.mean(W['temperature']))
temperature_rms = float(np.sqrt(np.mean((W['temperature']-temperature_mean)**2)))
wind_dir_mean = float(np.mean(W['wind_direction']))
wind_dir_rms = float(np.sqrt(np.mean((W['wind_direction']-wind_dir_mean)**2)))
rel_humid_mean = float(np.mean(W['rel_humidity']**2))
rel_humid_rms = float(np.sqrt(np.mean((W['rel_humidity']-rel_humid_mean)**2)) )
pressure_mean = float(np.mean(W['pressure']))
pressure_rms = float(np.sqrt(np.mean((W['pressure']-pressure_mean)**2)))
localtime = data[0].header['DATE-OBS']
totaltime = float(max(data[4].data['time']) )
PWV = float(data[0].header['PWV'])
PWVERR = float(data[0].header['PWVERR'])
q = 'INSERT INTO Overview(uid, localtime,totaltime, RestFreq, elevation, azimuth, source, fluxdens, wind_speed_mean, wind_speed_rms, wind_dir_mean, wind_dir_rms, pressure_mean, pressure_rms, rel_humid_mean, rel_humid_rms, temperature_mean, temperature_rms, pwv, pwv_rms) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) '
cur.execute(q,(uid,localtime,totaltime, RestFreq, elevation, azimuth, source, fluxdens, wind_speed_mean, wind_speed_rms, wind_dir_mean, wind_dir_rms, pressure_mean, pressure_rms, rel_humid_mean, rel_humid_rms, temperature_mean, temperature_rms, PWV, PWVERR))
# ANTENNA PARAMETERS
pwvdatafile = uid+'.pwv.fits'
pwvdata=fits.open(pwvdatafile)
Antpwv = pwvdata[1].data
Dpwv = pwvdata[4].data
Alpwv = pwvdata[5].data
antennanames = Antpwv['name'][:]
# find entries closest to timescale
mintimeind=[]
for ts in timescale:
mintimeind.append(np.argmin(abs(Alpwv['tscale']-ts)))
for antenna_name in antennanames:
pwv_mean = float(np.mean(Dpwv[antenna_name]))
pwv_rms = float(np.sqrt(np.mean((Dpwv[antenna_name]-pwv_mean)**2)))
adv_pwv_t1 = float(Alpwv[antenna_name][mintimeind[0]])
adv_pwv_t2 = float(Alpwv[antenna_name][mintimeind[1]])
adv_pwv_t3 = float(Alpwv[antenna_name][mintimeind[2]])
adv_pwv_t4 = float(Alpwv[antenna_name][mintimeind[3]])
adv_pwv_t5 = float(Alpwv[antenna_name][mintimeind[4]])
q = 'INSERT INTO Antenna(uid, antenna_name, pwv_mean,pwv_rms,adv_pwv_t1,adv_pwv_t2,adv_pwv_t3,adv_pwv_t4,adv_pwv_t5) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?) '
cur.execute(q,(uid, antenna_name, pwv_mean,pwv_rms,adv_pwv_t1,adv_pwv_t2,adv_pwv_t3,adv_pwv_t4,adv_pwv_t5))
# BASELINE PARAMETERS
A=data[1].data
D = data[2].data
RP = data[4].data
ARP = data[5].data
ACP = data[7].data
mintimeind=[]
for ts in timescale:
mintimeind.append(np.argmin(abs(ARP['tscale']-ts)))
viswvrdatafile = uid+'.viswvr.fits'
viswvrdata=fits.open(viswvrdatafile)
ARD = viswvrdata[5].data
baselinenames = D['name'][:]
lengths = D['xy'][:]
bl_dirs = get_bldir(A,baselinenames)
for idx,base in enumerate(baselinenames):
baseline_length=np.float(lengths[idx])
antbase=base.split('-')
baseline_dir=np.float(bl_dirs[idx])
adv_raw_t1 = float(ARP[base][mintimeind[0]])
adv_raw_t2 = float(ARP[base][mintimeind[1]])
adv_raw_t3 = float(ARP[base][mintimeind[2]])
adv_raw_t4 = float(ARP[base][mintimeind[3]])
adv_raw_t5 = float(ARP[base][mintimeind[4]])
adv_corr_t1 = float(ACP[base][mintimeind[0]])
adv_corr_t2 = float(ACP[base][mintimeind[1]])
adv_corr_t3 = float(ACP[base][mintimeind[2]])
adv_corr_t4 = float(ACP[base][mintimeind[3]])
adv_corr_t5 = float(ACP[base][mintimeind[4]])
adv_delay_t1 = float(ARD[base][mintimeind[0]])
adv_delay_t2 = float(ARD[base][mintimeind[1]])
adv_delay_t3 = float(ARD[base][mintimeind[2]])
adv_delay_t4 = float(ARD[base][mintimeind[3]])
adv_delay_t5 = float(ARD[base][mintimeind[4]])
q = 'INSERT INTO Baseline(uid,antenna1,antenna2,baseline_length,baseline_dir,adv_raw_t1,adv_raw_t2,adv_raw_t3,adv_raw_t4,adv_raw_t5,adv_corr_t1,adv_corr_t2,adv_corr_t3,adv_corr_t4,adv_corr_t5,adv_delay_t1,adv_delay_t2,adv_delay_t3,adv_delay_t4,adv_delay_t5) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
cur.execute(q,(uid,antbase[0],antbase[1],baseline_length,baseline_dir,adv_raw_t1,adv_raw_t2,adv_raw_t3,adv_raw_t4,adv_raw_t5,adv_corr_t1,adv_corr_t2,adv_corr_t3,adv_corr_t4,adv_corr_t5,adv_delay_t1,adv_delay_t2,adv_delay_t3,adv_delay_t4,adv_delay_t5))
con.commit()
except:
print "ERROR in putentry2", sys.exc_info()[0]
raise
[docs]def putentry3(results,database_name='',antenna_out='',baseline_out=''):
''' Enters the required values of a reduced asdm into the new style sql databas (comparenew.db)
:param results: UID fits file.
:param database_name: name of database to connect to and enter stats into.
:param antenna_out: list of antennas to be excluded. Separated by a comma.
:param baseline_out: list of baselines to exclude. syntax e.g. DV14-DA64,DV12-DA60,...
:type results: str
:type database_name: str
:type antenna_out: str
:type baseline_out: str
:returns: None
:rtype: None
'''
try:
antennas = [] # these are hard coded write as 'DAXX','DVXX'
flagantenna=antenna_out.split(',')
for flag in flagantenna: antennas.append(flag)
# flag out single baselines only
bad_baseline = [] # hardcoded if any
flagbaseline=baseline_out.split(',')
for flag in flagbaseline: bad_baseline.append(flag)
uidfull,data_type,suffix=results.split('.')
prefix,uid = uidfull.split('/')
data = fits.open(results)
W = data[3].data
print 'Excluding Antennas'
print antennas
print 'Excluding Baselines'
print bad_baseline
to_connect = '/lustre/allegro/allegro_staff/projects/allegro/wvr/postJan2015/masterdatabase/'+database_name
con = lite.connect(to_connect)
cur = con.cursor()
# connected - now test if the database exists
cur.execute("SELECT ? FROM sqlite_master WHERE type = ?",('Overview','table'))
data_table = cur.fetchone()
if data_table is None:
# if no database then create one
print ' no data base with that name - creating one'
# populate the datebase with tables, clearly it was a fresh opening
con.execute(''' create table Overview(
uid TEXT, localtime TEXT ,totaltime REAL,
RestFreq REAL, elevation REAL, azimuth REAL,
source TEXT, fluxdens REAL, wind_speed_mean REAL,
wind_speed_rms REAL, wind_dir_mean REAL, wind_dir_rms REAL,
pressure_mean REAL, pressure_rms REAL, rel_humid_mean REAL,
rel_humid_rms REAL, temperature_mean REAL, temperature_rms REAL,
pwv REAL, pwv_rms REAL);''')
con.execute(''' create table Antenna(
uid TEXT, antenna_name TEXT, pwv_mean REAL,
pwv_rms REAL, adv_pwv_t1 REAL, adv_pwv_t2 REAL,
adv_pwv_t3 REAL, adv_pwv_t4 REAL, adv_pwv_t5 REAL);''')
con.execute('''create table Baseline(
uid TEXT, antenna1 TEXT, antenna2 TEXT,
baseline_length REAL, baseline_dir REAL, adv_raw_t1 REAL,
adv_raw_t2 REAL, adv_raw_t3 REAL, adv_raw_t4 REAL,
adv_raw_t5 REAL, adv_corr_t1 REAL, adv_corr_t2 REAL,
adv_corr_t3 REAL, adv_corr_t4 REAL, adv_corr_t5 REAL,
adv_delay_t1 REAL, adv_delay_t2 REAL,adv_delay_t3 REAL,
adv_delay_t4 REAL,adv_delay_t5 REAL);''')
print '...'
print '......'
print '.........'
print '............new database made'
#get the header info
timescale = [1.1,12.,32.,64.,128.]
# ARRAY PARAMETERS
RestFreq = float(data[0].header['RESTFREQ'])
elevation = float(data[0].header['ALT'])
azimuth = float(data[0].header['AZ'])
source = data[0].header['SOURCE']
fluxdens = float(data[0].header['FLUXDENS'])
wind_speed_mean = float(np.mean(W['wind_speed']))
wind_speed_rms = float(np.sqrt(np.mean((W['wind_speed']-wind_speed_mean)**2)))
temperature_mean = float(np.mean(W['temperature']))
temperature_rms = float(np.sqrt(np.mean((W['temperature']-temperature_mean)**2)))
wind_dir_mean = float(np.mean(W['wind_direction']))
wind_dir_rms = float(np.sqrt(np.mean((W['wind_direction']-wind_dir_mean)**2)))
rel_humid_mean = float(np.mean(W['rel_humidity']))
rel_humid_rms = float(np.sqrt(np.mean((W['rel_humidity']-rel_humid_mean)**2)) )
pressure_mean = float(np.mean(W['pressure']))
pressure_rms = float(np.sqrt(np.mean((W['pressure']-pressure_mean)**2)))
localtime = data[0].header['DATE-OBS']
totaltime = float(max(data[4].data['time']) )
PWV = float(data[0].header['PWV'])
PWVERR = float(data[0].header['PWVERR'])
q = 'INSERT INTO Overview(uid, localtime,totaltime, RestFreq, elevation, azimuth, source, fluxdens, wind_speed_mean, wind_speed_rms, wind_dir_mean, wind_dir_rms, pressure_mean, pressure_rms, rel_humid_mean, rel_humid_rms, temperature_mean, temperature_rms, pwv, pwv_rms) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) '
cur.execute(q,(uid,localtime,totaltime, RestFreq, elevation, azimuth, source, fluxdens, wind_speed_mean, wind_speed_rms, wind_dir_mean, wind_dir_rms, pressure_mean, pressure_rms, rel_humid_mean, rel_humid_rms, temperature_mean, temperature_rms, PWV, PWVERR))
# ANTENNA PARAMETERS
pwvdatafile = prefix+'/'+uid+'.pwv.fits'
pwvdata=fits.open(pwvdatafile)
Antpwv = pwvdata[1].data
Dpwv = pwvdata[4].data
Alpwv = pwvdata[5].data
antennanames = Antpwv['name'][:]
# find entries closest to timescale
mintimeind=[]
for ts in timescale:
mintimeind.append(np.argmin(abs(Alpwv['tscale']-ts)))
for antenna_name in antennanames:
pwv_mean = float(np.mean(Dpwv[antenna_name]))
pwv_rms = float(np.sqrt(np.mean((Dpwv[antenna_name]-pwv_mean)**2)))
adv_pwv_t1 = float(Alpwv[antenna_name][mintimeind[0]])
adv_pwv_t2 = float(Alpwv[antenna_name][mintimeind[1]])
adv_pwv_t3 = float(Alpwv[antenna_name][mintimeind[2]])
adv_pwv_t4 = float(Alpwv[antenna_name][mintimeind[3]])
adv_pwv_t5 = float(Alpwv[antenna_name][mintimeind[4]])
if antenna_name not in antennas: # only fills if antenna is not listed as to flag
q = 'INSERT INTO Antenna(uid, antenna_name, pwv_mean,pwv_rms,adv_pwv_t1,adv_pwv_t2,adv_pwv_t3,adv_pwv_t4,adv_pwv_t5) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?) '
cur.execute(q,(uid, antenna_name, pwv_mean,pwv_rms,adv_pwv_t1,adv_pwv_t2,adv_pwv_t3,adv_pwv_t4,adv_pwv_t5))
# BASELINE PARAMETERS
A=data[1].data
D = data[2].data
RP = data[4].data # raw phase
ARP = data[5].data # raw phase tpd
ACP = data[7].data # corr phase tpf (6) is corr phase
mintimeind=[]
for ts in timescale:
mintimeind.append(np.argmin(abs(ARP['tscale']-ts)))
viswvrdatafile = prefix+'/'+uid+'.viswvr.fits'
viswvrdata=fits.open(viswvrdatafile)
ARD = viswvrdata[5].data
baselinenames = D['name'][:]
lengths = D['xy'][:]
bl_dirs = get_bldir(A,baselinenames)
for idx,base in enumerate(baselinenames):
baseline_length=np.float(lengths[idx])
antbase=base.split('-')
baseline_dir=np.float(bl_dirs[idx])
adv_raw_t1 = float(ARP[base][mintimeind[0]])
adv_raw_t2 = float(ARP[base][mintimeind[1]])
adv_raw_t3 = float(ARP[base][mintimeind[2]])
adv_raw_t4 = float(ARP[base][mintimeind[3]])
adv_raw_t5 = float(ARP[base][mintimeind[4]])
adv_corr_t1 = float(ACP[base][mintimeind[0]])
adv_corr_t2 = float(ACP[base][mintimeind[1]])
adv_corr_t3 = float(ACP[base][mintimeind[2]])
adv_corr_t4 = float(ACP[base][mintimeind[3]])
adv_corr_t5 = float(ACP[base][mintimeind[4]])
adv_delay_t1 = float(ARD[base][mintimeind[0]])
adv_delay_t2 = float(ARD[base][mintimeind[1]])
adv_delay_t3 = float(ARD[base][mintimeind[2]])
adv_delay_t4 = float(ARD[base][mintimeind[3]])
adv_delay_t5 = float(ARD[base][mintimeind[4]])
if antbase[0] not in antennas: # check for antenna flag in either
if antbase[1] not in antennas:
if base not in bad_baseline: # check for single bad baseline
q = 'INSERT INTO Baseline(uid,antenna1,antenna2,baseline_length,baseline_dir,adv_raw_t1,adv_raw_t2,adv_raw_t3,adv_raw_t4,adv_raw_t5,adv_corr_t1,adv_corr_t2,adv_corr_t3,adv_corr_t4,adv_corr_t5,adv_delay_t1,adv_delay_t2,adv_delay_t3,adv_delay_t4,adv_delay_t5) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
cur.execute(q,(uid,antbase[0],antbase[1],baseline_length,baseline_dir,adv_raw_t1,adv_raw_t2,adv_raw_t3,adv_raw_t4,adv_raw_t5,adv_corr_t1,adv_corr_t2,adv_corr_t3,adv_corr_t4,adv_corr_t5,adv_delay_t1,adv_delay_t2,adv_delay_t3,adv_delay_t4,adv_delay_t5))
con.commit()
con.close()
except:
print "ERROR in putentry3", sys.exc_info()[0]
raise