Source code for mascara.funcs.mio

# Copyright 2014 - 2015 Anna-Lea Lesage for MASCARA
#    This file is part of the mascara package.
#    mascara is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#    mascara is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    GNU General Public License for more details.
#    You should have received a copy of the GNU General Public License
#    along with mascara.  If not, see <>.

.. module:: mio
.. moduleauthor:: Anna-Lea Lesage   

I/O functions used for MASCARA

    - CheckFreeSpace() check how much disk space is free on computer
    - MakeDiskSpace() delete the oldest folder found in a given path
    - find_files(), find files in path according to root
    - list_directories(), list all directories 
    - pick_directories(), select from a list those who satisfy condition
    - makedir(), wrapper around os.makedirs to create several directories at once
    - cleanup() removes directories forcibly (even if data remains inside!)
    - moveData(), which is more a copy data than a move data
    - savepng()
    - savecompfits(), to save a compressed fits file

__version__ = '15.10.02'

#### File creating/modification/updating routines #### 
def readcfg(configfile, keys = '', separator = '='):
    ''' Reads a config file and returns the values for the given keywords
            - filename, the complete filename including path to read
            - keys, the key to read out of the file
            - separator, default is '=', what separates the key from
              the value in the file
            - None if the file or the keys is not found. Else the value 
              corresponding to the key
    import os
    if os.path.exists(configfile):
        with open(configfile, 'rb') as f:
            c =
        cl = c.split('\n')
        line = [l for l in cl if not l.startswith('#') if not l.strip()=='']
        for l in line:
            if l.find(keys) == 0:
                tmp = l.split(separator)[1].strip()
                if tmp.isalpha():
                    return tmp
                    return float(tmp)
        return None

def adapt_array(arr):
    ''' adapt_array is an adapter for sqlite allowing the storage of numpy arrays in
       a database-like format.
    from io import BytesIO
    import numpy as np
    out = BytesIO(), arr)
    return buffer(
def convert_array(text):
    ''' convert_array converts the informations contained in a sqlite database back 
       into numpy arrays. 
    import numpy as np
    from io import BytesIO
    out = BytesIO(text)
    return np.load(out)

[docs]def saveCompFits(path, name, data, header): ''' to save an image in a compressed fits. Inputs: - path, the path for saving the images - name, the name of the image - data, the data to be saved - header, the corresponding header The data is saved in a CompImageHDU, using a RICE compression type. If the data is not in UINT16 format, it will be converted before the compression. ''' import pyfits as pf import os ny, nx = data.shape #data = np.uint16(data) header.set('BITPIX', 16, comment='array data type', before=0) header.set('NAXIS', 2, comment='number of array dimensions', after='BITPIX') header.set('NAXIS1', nx, after='NAXIS') header.set('NAXIS2', ny, after='NAXIS1') f = pf.CompImageHDU(data=data, header=header, compression_type='HCOMPRESS_1', \ name='COMP_IMAGE', hcomp_scale=1., uint=True) f.verify(option='fix') f.writeto(os.path.join(path,name+'.fits'), clobber=True) return
[docs]def savepng(image, path, name, binfactor = 1): ''' savepng saves the input image at the given path. The image can be rebinned using the binfactor keyword. The scaling is defined via ... Inputs: - image, an array - path, the saving directory - the name of the file (without the .png) Keywords: - binfactor, if 1, the saved image has the same dimension as the original image. Set to larger as 1 to reduce image size. ''' import os from scipy.misc import imsave from mascara.funcs.utils import rebin binmage = rebin(image, binfactor) binmage = (binmage**0.1-1.5)/1.5*255 imsave(os.path.join(path, name+'.png'), binmage) return
[docs]def find_files(path, strm, format = '.fits'): """ List of the files in the given format in the given path directory. It works very similarly to : ls mydirectory/test*.fits Inputs: - path, the complete path of the directory to look at - strm, the string to find (using * to get the files) Keyword: - format, default '.fits' Output: - a list of all files satisfyig the condition """ import glob fn = sorted(glob.glob(path + str(strm) + format)) return fn
[docs]def list_directories(path, root): ''' list_directories works a bit list the os.listdir function, except that you can feed it a root and it'll list all directories having that root in their names in the given path. Inputs: - path, a string, giving the location where to look - root, a string, giving the root to use for search Output: - a list of the directories Example: >>> dires = list_directories('D:\\LaPalma\\', '20141114') ''' import os import subprocess import platform directories = [] if platform.system() == 'Windows': ps = subprocess.Popen("dir "+path+"*"+root+"* /B", shell=True, stdout=subprocess.PIPE) output = output = output.split('\r\n') for l in output: tmp = l.find(root) if tmp>=0: directories.append(path+l) return directories else: devnull = open(os.devnull,'wb') ps = subprocess.Popen("ls -d "+path+"*"+root+"*", shell=True, stdout=subprocess.PIPE, stderr=devnull) output = output = output.split('\n') devnull.close() if root != '': for l in output: if l.find(root)>0: directories.append(l) return directories else: return output
[docs]def pick_directories(dirlist, root): ''' picks the right directories for a list using the root as reference Inputs: - dirlist, a list of directories - root, a reference to search for in the list Output: - a list of the directories ''' if root: if len(root) >= 2: directory = [d for d in dirlist if d.find(root)>0] elif len(root) == 1: directory = [d for d in dirlist if d.endswith(root)] return directory
[docs]def check_free_space(folder= 'D:\\'): ''' CheckFreeSpace looks how much free space is in the given folder... preferably the hard disk. To be used to for overwriting/deleting old raw data at the beginning of the night, to avoid getting into memory troubles. Input: folder, a string giving the path of the folder to check Output: the amount of free space in MB ''' import ctypes as ct import platform import os if platform.system() == 'Windows': free_bytes = ct.c_ulonglong(0) ct.windll.kernel32.GetDiskFreeSpaceExW(ct.c_wchar_p(folder), None, None, \ ct.pointer(free_bytes)) return free_bytes.value/1024/1024 else: st = os.statvfs(folder) return st.f_bavail * st.f_frsize/1024/1024
[docs]def make_disk_space(path='D:\\'): ''' Removes the oldest files in the path Keyword: - path, to define where to remove files ''' import shutil import os filelist = [os.path.join(path,f) for f in os.listdir(path)] oldest = min(filelist, key=lambda x:os.stat(x).st_ctime) shutil.rmtree(oldest) return
def makedir(*args): ''' makedir is a subtitute for the function os.makedir. It creates several directories in the same root directory Inputs: - path, the path root for creating the directories - subpaths, the inside directoires Output: - none ''' import os path = args[0] if len(args)>1: for p in args[1:]: if not os.path.exists(p): os.makedirs(p) print 'Created directory %s' %p else: os.makedirs(path) return
[docs]def moveData(frompath, topath, rawpath = 'D:\\Raw\\', keeprawfiles = False): ''' Copy the Data from the directory frompath to the directory topath. and delete afterwards the frompath. Inputs: - frompath, the source directory - topath, the destination directory Keywords: - keeprawfiles, if True, the program makes a copy of the files from the source to the destination - rawpath, the directori ''' import os, shutil if keeprawfiles: os.makedirs(topath) shutil.copy(frompath, topath) else: for r in (os.path.join(frompath, j) for j in os.listdir(frompath)): if r.find('raw') > 0: if not os.path.exists(rawpath): os.makedirs(rawpath) shutil.copy(r, rawpath) else: if not os.path.exists(topath): os.makedirs(topath) shutil.copy(r, topath) return
def cleanup(*args, **kwargs): ''' Cleanup delete a complete directory tree Inputs: - path - logger (optional) Keyword: - force, if True, delete a directory even with files still in it ''' import shutil import os import numpy as np force = kwargs.get('force', False) if np.size(args) ==1: path = args[0] blati = os.listdir(path) if len(blati) >0: print "Some elements remained in {}".format(path) if not force: return if np.size(args) >1: path, qlog = args import logging import multiprocessing logger = logging.getLogger(multiprocessing.current_process().name) logger.setLevel(logging.INFO) blati = os.listdir(path) if len(blati) > 0: logger.warning('Some elements remained in tmp directory') shutil.rmtree(path) return