text: #!/usr/bin/env python3
  # -*- coding: utf-8 -*-
  """
  Created on Wed Jan 23 21:49:53 2019
  
  @author: kazuki.onodera
  
  NOT time series feature
  
  """
  
  import numpy as np
  import pandas as pd
  #from multiprocessing import Pool
  from sklearn.preprocessing import LabelEncoder
  
  import utils
  
  PREF = 'f002'
  
  
  col_cat = [
           'ProductName',                              # cardinality(6, 6), same distribuyion
           'EngineVersion',                           # maybe time series
           'AppVersion',                              # maybe time series
           'AvSigVersion',                            # maybe time series
           'DefaultBrowsersIdentifier',               # cardinality(1730, 1548)
           'AVProductStatesIdentifier',               # cardinality(28970, 23492)
           'CountryIdentifier',                        # cardinality(222, 222)
           'CityIdentifier',                           # cardinality(107366, 105817)
           'OrganizationIdentifier',                   # cardinality(49, 50)
           'GeoNameIdentifier',                        # cardinality(292, 289)
           'LocaleEnglishNameIdentifier',              # cardinality(276, 278)
           'Platform',                                 # cardinality(4, 4)
           'Processor',                                # cardinality(3, 3)
           'OsVer',                                   # cardinality(58, 44) maybe time series
           'OsPlatformSubRelease',                     # cardinality(58, 44)
           'OsBuildLab',                              # cardinality(663, 673)
           'SkuEdition',                               # cardinality(8, 8)
           'IeVerIdentifier',                         # cardinality(303, 294) maybe time series
           'SmartScreen',                              # cardinality(21, 21)
           'Census_MDC2FormFactor',                    # cardinality(13, 14)
           'Census_DeviceFamily',                      # cardinality(3, 3)
           'Census_OEMNameIdentifier',                # cardinality(3832, 3685)
           'Census_OEMModelIdentifier',               # cardinality(175365, 167776)
           'Census_ProcessorManufacturerIdentifier',   # cardinality(7, 7)
           'Census_ProcessorModelIdentifier',          # cardinality(3428, 3438)
           'Census_ProcessorClass',                    # cardinality(3, 3)
           'Census_PrimaryDiskTypeName',               # cardinality(4, 4)
           'Census_ChassisTypeName',                   # cardinality(52, 48)
           'Census_PowerPlatformRoleName',             # cardinality(10, 10)
           'Census_InternalBatteryType',               # cardinality(78, 63)
           'Census_OSVersion',                        # cardinality(469, 475)
           'Census_OSArchitecture',                    # cardinality(3, 3)
           'Census_OSBranch',                          # cardinality(32, 29)
           'Census_OSEdition',                         # cardinality(33, 36)
           'Census_OSSkuName',                         # cardinality(30, 31)
           'Census_OSInstallTypeName',                 # cardinality(9, 9)
           'Census_OSInstallLanguageIdentifier',       # cardinality(39, 39)
           'Census_OSUILocaleIdentifier',              # cardinality(147, 139)
           'Census_OSWUAutoUpdateOptionsName',         # cardinality(6, 6)
           'Census_GenuineStateName',                  # cardinality(5, 5)
           'Census_ActivationChannel',                 # cardinality(6, 6)
           'Census_FlightRing',                        # cardinality(10, 11)
           'Census_FirmwareManufacturerIdentifier',    # cardinality(712, 722)
           'Census_FirmwareVersionIdentifier',         # cardinality(50494, 49811)
           'Wdft_RegionIdentifier',                    # cardinality(15, 15)
           
           'OsBuild',                                  # cardinality(58, 44)
           'OsSuite',                                  # cardinality(58, 44)
           
           'OsBuildLab_major',
           'OsBuildLab_minor',
           'OsBuildLab_build',
           'OsBuildLab_architecture',
           ]
  
  
  
  # =============================================================================
  # main
  # =============================================================================
  if __name__ == "__main__":
      utils.start(__file__)
      
      tr = pd.read_feather('../data/train.f')[col_cat]
      te = pd.read_feather('../data/test.f')[col_cat]
      
      
      le = LabelEncoder()
      
      for c in col_cat:
          print('processing', c)
          
          if tr[c].dtype != 'O':
              min_ = min(tr[c].min(), te[c].min()) -1
              tr[c].fillna(min_, inplace=True)
              te[c].fillna(min_, inplace=True)
          else:
              tr[c].fillna('na dayo', inplace=True)
              te[c].fillna('na dayo', inplace=True)
          le.fit( tr[c].append(te[c]) )
          tr[c] = le.transform(tr[c])
          te[c]  = le.transform(te[c])
      
      utils.reduce_mem_usage(tr)
      utils.reduce_mem_usage(te)
      
      tr.add_prefix(PREF+'_').to_feather(f'../data/train_{PREF}.f')
      te.add_prefix(PREF+'_').to_feather(f'../data/test_{PREF}.f')
      
      
      print('==== categories ====')
      print(tr.head().add_prefix(PREF+'_').columns.tolist())
      
      utils.end(__file__)
  