# Copyright 2014 - 2015 Anna-Lea Lesage for MASCARA
#
# This file is part of the mascara package.
# mascara is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# mascara is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with mascara. If not, see <http://www.gnu.org/licenses/>.
'''
.. module:: mio
.. moduleauthor:: Anna-Lea Lesage
I/O functions used for MASCARA
Includes:
- CheckFreeSpace() check how much disk space is free on computer
- MakeDiskSpace() delete the oldest folder found in a given path
- find_files(), find files in path according to root
- list_directories(), list all directories
- pick_directories(), select from a list those who satisfy condition
- makedir(), wrapper around os.makedirs to create several directories at once
- cleanup() removes directories forcibly (even if data remains inside!)
- moveData(), which is more a copy data than a move data
- savepng()
- savecompfits(), to save a compressed fits file
'''
__version__ = '15.10.02'
########################################################
#### File creating/modification/updating routines ####
def readcfg(configfile, keys = '', separator = '='):
''' Reads a config file and returns the values for the given keywords
Inputs:
- filename, the complete filename including path to read
Keywords:
- keys, the key to read out of the file
- separator, default is '=', what separates the key from
the value in the file
Output:
- None if the file or the keys is not found. Else the value
corresponding to the key
'''
import os
if os.path.exists(configfile):
with open(configfile, 'rb') as f:
c = f.read()
cl = c.split('\n')
line = [l for l in cl if not l.startswith('#') if not l.strip()=='']
for l in line:
if l.find(keys) == 0:
tmp = l.split(separator)[1].strip()
if tmp.isalpha():
return tmp
else:
return float(tmp)
else:
return None
def adapt_array(arr):
''' adapt_array is an adapter for sqlite allowing the storage of numpy arrays in
a database-like format.
'''
from io import BytesIO
import numpy as np
out = BytesIO()
np.save(out, arr)
out.seek(0)
return buffer(out.read())
def convert_array(text):
''' convert_array converts the informations contained in a sqlite database back
into numpy arrays.
'''
import numpy as np
from io import BytesIO
out = BytesIO(text)
out.seek(0)
return np.load(out)
[docs]def saveCompFits(path, name, data, header):
''' to save an image in a compressed fits.
Inputs:
- path, the path for saving the images
- name, the name of the image
- data, the data to be saved
- header, the corresponding header
The data is saved in a CompImageHDU, using a RICE compression type.
If the data is not in UINT16 format, it will be converted before
the compression.
'''
import pyfits as pf
import os
ny, nx = data.shape
#data = np.uint16(data)
header.set('BITPIX', 16, comment='array data type', before=0)
header.set('NAXIS', 2, comment='number of array dimensions', after='BITPIX')
header.set('NAXIS1', nx, after='NAXIS')
header.set('NAXIS2', ny, after='NAXIS1')
f = pf.CompImageHDU(data=data, header=header, compression_type='HCOMPRESS_1', \
name='COMP_IMAGE', hcomp_scale=1., uint=True)
f.verify(option='fix')
f.writeto(os.path.join(path,name+'.fits'), clobber=True)
return
[docs]def savepng(image, path, name, binfactor = 1):
''' savepng saves the input image at the given path. The image can be rebinned
using the binfactor keyword. The scaling is defined via ...
Inputs:
- image, an array
- path, the saving directory
- the name of the file (without the .png)
Keywords:
- binfactor, if 1, the saved image has the same dimension as the original
image. Set to larger as 1 to reduce image size.
'''
import os
from scipy.misc import imsave
from mascara.funcs.utils import rebin
binmage = rebin(image, binfactor)
binmage = (binmage**0.1-1.5)/1.5*255
imsave(os.path.join(path, name+'.png'), binmage)
return
[docs]def find_files(path, strm, format = '.fits'):
""" List of the files in the given format in the given path directory.
It works very similarly to :
ls mydirectory/test*.fits
Inputs:
- path, the complete path of the directory to look at
- strm, the string to find (using * to get the files)
Keyword:
- format, default '.fits'
Output:
- a list of all files satisfyig the condition
"""
import glob
fn = sorted(glob.glob(path + str(strm) + format))
return fn
[docs]def list_directories(path, root):
''' list_directories works a bit list the os.listdir function, except that you
can feed it a root and it'll list all directories having that root in
their names in the given path.
Inputs:
- path, a string, giving the location where to look
- root, a string, giving the root to use for search
Output:
- a list of the directories
Example:
>>> dires = list_directories('D:\\LaPalma\\', '20141114')
'''
import os
import subprocess
import platform
directories = []
if platform.system() == 'Windows':
ps = subprocess.Popen("dir "+path+"*"+root+"* /B", shell=True, stdout=subprocess.PIPE)
output = ps.stdout.read()
output = output.split('\r\n')
for l in output:
tmp = l.find(root)
if tmp>=0:
directories.append(path+l)
return directories
else:
devnull = open(os.devnull,'wb')
ps = subprocess.Popen("ls -d "+path+"*"+root+"*", shell=True, stdout=subprocess.PIPE, stderr=devnull)
output = ps.stdout.read()
output = output.split('\n')
devnull.close()
if root != '':
for l in output:
if l.find(root)>0:
directories.append(l)
return directories
else:
return output
[docs]def pick_directories(dirlist, root):
''' picks the right directories for a list using the root as reference
Inputs:
- dirlist, a list of directories
- root, a reference to search for in the list
Output:
- a list of the directories
'''
if root:
if len(root) >= 2:
directory = [d for d in dirlist if d.find(root)>0]
elif len(root) == 1:
directory = [d for d in dirlist if d.endswith(root)]
return directory
[docs]def check_free_space(folder= 'D:\\'):
''' CheckFreeSpace looks how much free space is in the given folder...
preferably the hard disk.
To be used to for overwriting/deleting old raw data at the beginning of
the night, to avoid getting into memory troubles.
Input: folder, a string giving the path of the folder to check
Output: the amount of free space in MB
'''
import ctypes as ct
import platform
import os
if platform.system() == 'Windows':
free_bytes = ct.c_ulonglong(0)
ct.windll.kernel32.GetDiskFreeSpaceExW(ct.c_wchar_p(folder), None, None, \
ct.pointer(free_bytes))
return free_bytes.value/1024/1024
else:
st = os.statvfs(folder)
return st.f_bavail * st.f_frsize/1024/1024
[docs]def make_disk_space(path='D:\\'):
''' Removes the oldest files in the path
Keyword:
- path, to define where to remove files
'''
import shutil
import os
filelist = [os.path.join(path,f) for f in os.listdir(path)]
oldest = min(filelist, key=lambda x:os.stat(x).st_ctime)
shutil.rmtree(oldest)
return
def makedir(*args):
''' makedir is a subtitute for the function os.makedir.
It creates several directories in the same root directory
Inputs:
- path, the path root for creating the directories
- subpaths, the inside directoires
Output:
- none
'''
import os
path = args[0]
if len(args)>1:
for p in args[1:]:
if not os.path.exists(p):
os.makedirs(p)
print 'Created directory %s' %p
else:
os.makedirs(path)
return
[docs]def moveData(frompath, topath, rawpath = 'D:\\Raw\\', keeprawfiles = False):
''' Copy the Data from the directory frompath to the directory topath.
and delete afterwards the frompath.
Inputs:
- frompath, the source directory
- topath, the destination directory
Keywords:
- keeprawfiles, if True, the program makes a copy of the files
from the source to the destination
- rawpath, the directori
'''
import os, shutil
if keeprawfiles:
os.makedirs(topath)
shutil.copy(frompath, topath)
else:
for r in (os.path.join(frompath, j) for j in os.listdir(frompath)):
if r.find('raw') > 0:
if not os.path.exists(rawpath):
os.makedirs(rawpath)
shutil.copy(r, rawpath)
else:
if not os.path.exists(topath):
os.makedirs(topath)
shutil.copy(r, topath)
return
def cleanup(*args, **kwargs):
''' Cleanup delete a complete directory tree
Inputs:
- path
- logger (optional)
Keyword:
- force, if True, delete a directory even with files still in it
'''
import shutil
import os
import numpy as np
force = kwargs.get('force', False)
if np.size(args) ==1:
path = args[0]
blati = os.listdir(path)
if len(blati) >0:
print "Some elements remained in {}".format(path)
if not force:
return
if np.size(args) >1:
path, qlog = args
import logging
import multiprocessing
logger = logging.getLogger(multiprocessing.current_process().name)
logger.setLevel(logging.INFO)
blati = os.listdir(path)
if len(blati) > 0:
logger.warning('Some elements remained in tmp directory')
shutil.rmtree(path)
return
