#!/usr/local/bin/python
#
# Script to understand filesystem details
#
# Written by Dougal Scott <dwagon@pobox.com>
# $Id: filesys.py 4431 2013-02-27 07:38:45Z dougals $
# $HeadURL: http://svn/ops/unix/explorer/trunk/explorer/filesys.py $

import os
import sys
import getopt
import re
import explorerbase
import storage

##########################################################################
# Filesystem #############################################################
##########################################################################


class Filesystem(explorerbase.ExplorerBase):
    ##########################################################################

    def __init__(self, config, fs, data, alldata):
        self.objname = fs
        explorerbase.ExplorerBase.__init__(self, config)
        self.data = data
        self.alldata = alldata

    ##########################################################################
    def analyse(self):
        if not self['protected']:
            self.addIssue(
                'unprotected', obj=self.name(), text="%s is not redundant" % self.name())

    ##########################################################################
    def getNotes(self):
        return self['describer']

##########################################################################
# Filesystems ############################################################
##########################################################################


class Filesystems(explorerbase.ExplorerBase):

    def __init__(self, config):
        explorerbase.ExplorerBase.__init__(self, config)
        self.st = storage.Storage(config)
        for fs in self.st.keys():
            if '_type' in self.st[fs] and self.st[fs]['_type'] == 'filesystem':
                self[fs] = Filesystem(config, fs, self.st[fs], self.st)
        self.analyse()

    ##########################################################################
    def analyse(self):
        for fs in self.fsList():
            fs.analyse()
            self.inheritIssues(fs)

    ##########################################################################
    def fsList(self):
        return [self[fs] for fs in sorted(self.keys())]

    ##########################################################################
    def capacity(self):
        used = 0
        capac = 0
        for mp in self.fsList():
            if mp['fstype'] in ("zfs", "nfs"):
                continue
            if 'kbytes' in mp:
                capac += mp['kbytes']
            if 'used' in mp:
                used += mp['used']
        return (used, capac)

##########################################################################
# storageFilesystems #####################################################
##########################################################################


class storageFilesystems(explorerbase.ExplorerBase):

    """Understand explorer output with respect to all filesystems and their ilk
    """

    ##########################################################################

    def __init__(self, config, data={}):
        explorerbase.ExplorerBase.__init__(self, config)
        self.data = data
        self.parse()

    ##########################################################################
    def parse(self):
        self['filesystems'] = set()
        self.parseMount()
        self.parseFstab()
        self.parseDf()
        self.virtualCheck()

    ##########################################################################
    def parseDf(self):
        """ Provide to the filesystem class:
                device - device filesystem is mount on
                kbytes - total capacity in kbytes
                used - space used up in kbytes
                avail - space left to use in kbytes
                pct - % used
        """

        if self.config['explorertype'] == 'solaris':
            dffile = None
            for fname in ['disks/df-klZ.out', 'disks/df-kl.out', 'disks/df-k.out']:
                if self.exists(fname):
                    dffile = fname
                    break
            if not dffile:
                self.Warning("No usable df file")
                return
        elif self.config['explorertype'] == 'linux':
            dffile = 'df'
        else:
            self.Fatal("parseDf - unsupported explorer type %s" %
                       self.config['explorertype'])
        self.parseDf_real(dffile)

    ##########################################################################
    def virtualCheck(self):
        """ Go through all the filesystems and calculate whether they are virtual
        filesystems or not
        """
        for fs in self['filesystems'].copy():
            if self.isVirtual(fs):
                del self[fs]
                self['filesystems'].remove(fs)
            else:
                self[fs]['description'] = 'Filesystem'

    ##########################################################################
    def isVirtual(self, fs):
        # mvfs - Multi Version File System - ClearCase
        # odm - Oracle Disk Manager
        if 'fstype' not in self[fs]:
            return False
        if self[fs]['fstype'] in ('mntfs', 'proc', 'fd', 'devfs',
                                  'objfs', 'ctfs', 'tmpfs', 'lofs', 'rpc_pipefs',
                                  'devpts', 'usbfs', 'binfmt_misc', 'sysfs', 'sharefs',
                                  'iso9660', 'usbdevfs', 'hsfs', 'oracleasmfs', 'shmfs',
                                  'shm', 'mvfs', 'autofs', 'odm',
                                  #'nfs', 'nfsd',
                                  ):
            return True
        if '/libc_' in fs:
            return True
        if 'device' in self[fs] and self[fs]['device'] == 'mnttab':
            return True
        return False

    ##########################################################################
    def dehumanise(self, str):
        """ Convert a string generated by a -h option (e.g. 37M, 1.7G) into
        kbytes
        """
        num = float(str[:-1])
        units = str[-1]
        if units == 'G':
            return int(num * 1024 * 1024)
        if units == 'M':
            return int(num * 1024)
        self.Fatal("dehumanise - unknown units %s" % units)

    ##########################################################################
    def parseDf_real(self, dffile):
        f = self.open(dffile)
        oldline = None
        for line in f:
            if line.startswith('Filesystem'):
                continue
            if line.startswith('/bin/df'):
                continue
            if oldline and line.startswith(' '):
                line = "%s %s" % (oldline, line)
                oldline = None
            bits = line.split()
            if len(bits) == 1:
                oldline = line
                continue
            mp = bits[-1]
            if mp not in self.data:
                continue
            fs = self[mp]
            if not 'device' in fs or not fs['device']:
                fs['device'] = bits[0]
            try:
                fs['kbytes'] = int(bits[1])       # Capacity
            except ValueError:
                fs['kbytes'] = self.dehumanise(bits[1])
            try:
                fs['used'] = int(bits[2])  # K used
            except ValueError:
                fs['used'] = self.dehumanise(bits[2])
            try:
                fs['avail'] = int(bits[3])        # K available
            except ValueError:
                fs['avail'] = self.dehumanise(bits[3])

            fs['pct'] = bits[4]
        f.close()

    ##########################################################################
    def fsList(self):
        return self['filesystems']

    ##########################################################################
    def parseMount(self):
        """ Parse the mounted filesystems
        Need to create new filesystems if found and set the following attributes
                device - the physical device used
                fstype - the type of filesystem (ufs, ext3, etc)
        """
        if self.config['explorertype'] == 'solaris':
            self.parseSolaris_mount()
            self.parseSolaris_mnttab()
        elif self.config['explorertype'] == 'linux':
            self.parseLinux_mount()
        else:
            self.Fatal("parseMount - unsupported explorer type %s" %
                       self.config['explorertype'])

    ##########################################################################
    def parseLinux_mount(self):
        f = self.open('mount')
        for line in f:
            line = line.strip()
            if line == '/bin/mount':
                continue
            m = re.search(
                '(?P<dev>\S+) on (?P<mp>\S+) type (?P<fstype>\S+) \((?P<opts>.*)\)', line)
            mp = m.group('mp')
            dev = self.sanitiseDevice(m.group('dev'))
            self[mp] = storage.Storage.initialDict(
                {'_type': 'filesystem', 'usepoint': '', '_origin': 'mount'})
            self['filesystems'].add(mp)
            self[mp]['fstype'] = m.group('fstype')
            if self[mp]['fstype'] == 'nfs':
                self[mp]['protected'] = True
            self[mp]['device'] = dev
            self[mp]['usepoint'] = dev
            self[mp]['contains'].add(dev)
            self[mp]['use'].add(mp)
            self[mp]['opts'] = m.group('opts')
        f.close()

    ##########################################################################
    def parseSolaris_mnttab(self):
        filename = 'etc/mnttab'
        if not self.exists(filename):
            return
        f = self.open(filename)
        for line in f:
            if ':vold' in line:
                continue
            bits = line.split()
            mp = bits[1]
            if mp not in self:
                self[mp] = storage.Storage.initialDict(
                    {'_type': 'filesystem', '_origin': filename})
                self['filesystems'].add(mp)
                self[mp]['device'] = bits[0]
            if 'fstype' not in self[mp] or not self[mp]['fstype']:
                self[mp]['fstype'] = bits[2]
            if self[mp]['fstype'] == 'nfs':
                self[mp]['device'] = bits[0]
                self[mp]['protected'] = 'NFS'
        f.close()

    ##########################################################################
    def parseSolaris_mount(self):
        filename = None
        for fname in ('disks/mount-v.out', 'disks/mount.out'):
            if self.exists(fname):
                filename = fname
                break
        if not filename:
            self.Warning("No usable mount output")
            return
        f = self.open(filename)
        for line in f:
            if line.find('zone=') >= 0:
                continue
            bits = line.split()
            # If the mount-v is unavailable, then this isn't what we want
            if '-v' in filename:
                mp = bits[2]
                if mp not in self:
                    self[mp] = storage.Storage.initialDict(
                        {'_type': 'filesystem', 'usepoint': '', '_origin': filename})
                    self['filesystems'].add(mp)
                fs = self[mp]
                fs['device'] = self.sanitiseDevice(bits[0])
                fs['usepoint'] = fs['device']
                fs['fstype'] = bits[4]
                fs['contains'].add(fs['device'])
            else:
                mp = bits[1]
                if mp not in self:
                    self[mp] = storage.Storage.initialDict(
                        {'_type': 'filesystem', 'usepoint': '', '_origin': filename, 'fstype': ''})
                    self['filesystems'].add(mp)
                fs = self[mp]
                fs['device'] = self.sanitiseDevice(bits[0])
                fs['usepoint'] = fs['device']
                fs['fstype'] = bits[2]
                fs['contains'].add(fs['device'])
                fs['use'].add(mp)
        f.close()

    ##########################################################################
    def parseFstab(self):
        try:
            if self.config['explorertype'] == 'solaris':
                self.parseSolaris_vfstab()
            elif self.config['explorertype'] == 'linux':
                self.parseLinux_fstab()
            else:
                self.Fatal("parseFstab - unsupported explorer type %s" %
                           self.config['explorertype'])
        except UserWarning, err:
            self.Warning(err)

    ##########################################################################
    def parseLinux_fstab(self):
        # TODO - currently ignored
        f = self.open('etc/fstab')
        for line in f:
            line = line.strip()
            if not line or line.startswith('#'):
                continue
            bits = line.split()
            dev = bits[0]
            mp = bits[1]
            fstype = bits[2]
            try:
                opts = bits[3]
            except IndexError:
                if fstype != 'nfs':
                    self.Warning("Malformed fstab line: %s" % line)
        f.close()

    ##########################################################################
    def parseSolaris_vfstab(self):
        """
        Analyse vfstab output
        Only do real filesystems, no swap
        """
        f = self.open('etc/vfstab')
        for line in f:
            line = line.rstrip()
            if not line or line.startswith('#'):
                continue
            bits = line.split()
            if len(bits) < 4:
                self.Warning("Unhandled vfstab line: %s" % line)
                continue
            mp = bits[2]
            if mp == '-':
                continue
            # Have seen this which causes problems
            if mp.endswith('/') and mp != '/':
                mp = mp[:-1]
            if '/dsk/' in bits[1]:      # Should always be /rdsk/
                self.addIssue(
                    'vfstab', obj=mp, text="FSCK device misconfigured for %s" % mp)
            if mp not in self:
                self.addConcern(
                    "vfstab", obj=mp, text="Mounted filesystem is not in vfstab")
                self[mp] = storage.Storage.initialDict(
                    {'_type': 'filesystem', '_origin': 'etc/vfstab'})
            fs = self[mp]
            newdev = self.sanitiseDevice(bits[0])
            if 'device' in fs and fs['device'] != newdev:
                self.addIssue("vfstab", obj=mp, text="Mounted device %s disagrees with vfstab device %s" % (
                    fs['device'], newdev))
            fs['device'] = self.sanitiseDevice(bits[0])
            fs['usepoint'] = fs['device']
            fs['use'].add(mp)
            fs['contains'].add(fs['device'])
            fs['fstype'] = bits[3]
            self['filesystems'].add(mp)
            self[mp] = fs
        f.close()

    ##########################################################################
        # TODO: Add code to add up filesystem size +  cap
        # TODO: access zfs details to get userd + kbytes.
    def crossPopulate(self, data):
        # TODO - filesystems that we only know from the cluster resourcing
        for cfs in self['clusterfs']:
            if cfs not in self:
                self.Debug("Thinking about adding cluster fs=%s" % cfs)
                #self[cfs]=storage.Storage.initialDict({'_type': 'filesystem', 'usepoint':'', 'cluster':True})
            else:
                self[cfs]['cluster'] = True
        for fs in self.fsList():
            for dev in data[fs]['devices']:
                if dev in data:
                    data[dev]['use'].add(fs)

                    # If the device is protected then the filesytem on it is
                    # also protected
                    if 'protected' in data[dev]:
                        data[fs]['protected'] = data[dev]['protected']
                        #self.Warning("Filesystem %s is protected (%s) as it is on %s" % (fs, data[dev]['protected'], dev))
                else:
                    self.Warning(
                        "FS %s relies on non existant device %s" % (fs, dev))

# EOF
