text:
<repo_name>f-me/vin-parser<file_sep>/vin-parser.cabal
Name:                vin-parser
Version:             0.7.1.1
Synopsis:            Library for importing vendor-specific VIN files to CaRMa.
License:             BSD3
Author:              Alexandr `voidex` Ruchkin
Maintainer:          <EMAIL>

Category:            Database
Build-type:          Simple

Cabal-version:       >= 1.8
Tested-with:         GHC == 7.6.1

Library
  hs-source-dirs: src
  Exposed-Modules:
    Snap.Snaplet.Vin
    Vin.Field
    Vin.Model
    Vin.Models
    Vin.Models.Cars
    Vin.Utils
    Vin.Load
    Vin.Store
    Vin.Import

  Other-Modules:

  Build-depends:
    aeson                       == 0.6.*,
    base                        >= 4 && < 5,
    bytestring                  == 0.10.*,
    carma-http                  >= 0.3 && < 0.5,
    conduit                     == 1.0.*,
    containers                  >= 0.4 && < 0.6,
    csv-conduit                 == 0.5.*,
    directory                   == 1.2.*,
    filepath                    == 1.3.*,
    ghc-prim                    == 0.3.*,
    email-validate              == 0.2.*,
    lens                        >= 3.8 && < 3.11,
    mmorph                      == 1.0.*,
    MonadCatchIO-transformers   >= 0.2 && < 0.4,
    mtl                         >= 2 && < 3,
    deepseq                     == 1.3.*,
    old-locale                  >= 1.0 && < 1.1,
    resourcet                   == 0.4.*,
    snap                        >= 0.9 && < 0.14,
    snap-core                   == 0.9.*,
    snap-server                 == 0.9.*,
    text                        >= 0.11 && < 0.12,
    text-icu                    == 0.6.*,
    time                        >= 1.4 && < 1.5,
    transformers                >= 0.2 && < 0.4,
    unordered-containers        == 0.2.*,
    xlsx                        >= 0.0.2.1 && < 0.1

  ghc-options: -Wall -fno-warn-unused-do-bind

source-repository head
  type:     git
  location: git://github.com/f-me/vin-parser.git
<file_sep>/src/Vin/Models.hs
{-# LANGUAGE OverloadedStrings #-}

module Vin.Models (
    runWithDicts, runDict,
    models
    ) where

import Control.Arrow
import Control.Applicative
import Control.Monad
import Control.Monad.Reader
import Data.Aeson
import Data.Char (toUpper, toLower)
import Data.List (find)
import Data.Maybe (listToMaybe, fromMaybe, mapMaybe)
import qualified Data.Map as M
import Data.Text (Text)
import qualified Data.Text as T
import Data.Traversable (sequenceA)

import Vin.Field
import Vin.Model
import Vin.Models.Cars

-- | Dictionary with car models
dictionaries :: FilePath
dictionaries = "resources/site-config/dictionaries/CarModels.json"

-- | Dictionary with car makers
makers :: FilePath
makers = "resources/site-config/dictionaries/CarMakers.json"

-- | Dictionary with formats
formats :: FilePath
formats = "resources/site-config/dictionaries/VinFormats.json"

-- | Dictionary with colors
colors :: FilePath
colors = "resources/site-config/dictionaries/Colors.json"

-- | Dictionaries loaded from JSON
data CarDictionaries = CarDictionaries {
    makersDict :: M.Map Text Text,
    -- ^ @Map@ from maker label to maker value
    modelsDict :: M.Map Text (M.Map Text Text),
    -- ^ @Map@ from maker value to @Map@ from model label to model value
    formatsList :: [String],
    -- ^ List of formats
    colorsDict :: M.Map Text Text }
    -- ^ @Map@ from color label to color value

-- | Dictionaried
type Dict a = Reader CarDictionaries a

-- | Load subdictionary from @Value@
getSubMap :: Value -> [String] -> M.Map Text Text
getSubMap v sub = maybe M.empty (M.unions . map mk) $ getMember v ("entries" : sub) where
    mk e = M.singleton (value e) (label e)

-- | Get top-level dictionary from @Value@
getMap :: Value -> M.Map Text Text
getMap v = getSubMap v []

-- | Make @Map@ from value to key
reverseMap :: (Ord k, Ord a) => M.Map k a -> M.Map a k
reverseMap = M.fromList . map (snd &&& fst) . M.toList

-- | Load dicts and run action
runWithDicts :: FilePath -> FilePath -> FilePath -> FilePath -> Dict a -> IO (Maybe a)
runWithDicts fm fc fp fcl act = do
    m <- fmap (fmap (reverseMap . getMap)) $ loadValue fm
    c <- loadValue fc
    let
        getCarModels :: Value -> Text -> M.Map Text (M.Map Text Text)
        getCarModels v make = M.singleton make (reverseMap $ getSubMap v [T.unpack make])
        mdls = do
            m' <- m
            c' <- c
            return $ M.unions $ map (getCarModels c') (M.elems m')
    p <- fmap (fmap (map T.unpack . M.keys . getMap)) $ loadValue fp
    cl <- fmap (fmap (reverseMap . getMap)) $ loadValue fcl
    return $ fmap (runReader act) $ liftM4 CarDictionaries m mdls p cl

-- | Run with default dictionaries
runDict :: Dict a -> IO (Maybe a)
runDict = runWithDicts makers dictionaries formats colors

-- | Get car models @Map@ from label to value for maker specified
getCarModels :: Text -> Dict (M.Map Text Text)
getCarModels make = do
    mdls <- asks modelsDict
    maybe (return M.empty) return $ M.lookup make mdls

-- | Get make field parser and model field parser
makeModelParser :: Dict (String -> Field Text, String -> String -> Field Text)
makeModelParser = do
    makers <- asks makersDict
    models <- asks modelsDict
    return (tableField makers, modelField makers models)

-- | Make model by its name
models :: Dict (String -> Maybe Model)
models = do
    m <- sequence [
        ford,
        fordPlus,
        vwMotor,
        vwCommercial,
        opel,
        hummer,
        chevroletNAO,
        chevroletKorea,
        cadillac,
        vwRuslan,
        chartis,
        vwAvilon,
        atlantM,
        autocraft,
        europlan,
        b2c,
        vtb24,
        citroenPeugeot "citroen",
        citroenPeugeot "peugeot"]
    u <- universal
    fs <- asks formatsList
    return $ \p -> find ((== p) . modelProgram) m <|> fmap u (find (== p) fs)

-- Predefined fields

car_buyDate = ("carBuyDate", time)
car_checkupDate = ("carCheckupDate", time)
car_checkupMileage = ("carCheckupMilage", int)
car_color = ("carColor", text)
car_make = ("carMake", text)
car_makeYear = ("carMakeYear", int)
car_model = ("carModel", text)
car_plateNum = ("carPlateNum", text)
car_seller = ("carSeller", text)
car_transmission = ("carTransmission", text)
car_vin = ("carVin", vin)
car_warrantyEnd = ("warrantyEnd", time)
car_warrantyStart = ("warrantyStart", time)
cardNumber = ("cardNumber", text)
cardOwner = ("cardOwner", text)
manager = ("manager", text)
milageTO = ("milageTO", int)
program = ("program", text)
validFrom = ("contractValidFromDate", time)
validUntil = ("contractValidUntilDate", time)
validUntilMilage = ("contractValidUntilMilage", int)

-- Fields not present in contract model
-- car_motor = ("car_motor", text)
modelCode = ("modelCode", text)
ownerEmail = ("contact_ownerEmail", email)
ownerName = ("contact_ownerName", text)
ownerPhone = ("contact_ownerPhone1", text)
serviceInterval = ("cardNumber_serviceInterval", int)

-- Helpers

-- | Words from several fields
wordsFrom :: [String] -> Field Text
wordsFrom = fmap T.unwords . sequenceA . map tryField

-- | Capitalize text
capitalized :: Text -> Text
capitalized str = case T.uncons str of
    Nothing -> T.empty
    Just (h, t) -> T.cons (toUpper h) (T.map toLower t)

-- | Take first word
firstWord :: Field Text -> Field Text
firstWord fld = fld >>= maybe empty return . listToMaybe . T.words

-- | Take second word
secondWord :: Field Text -> Field Text
secondWord fld = fld >>= maybe empty return . listToMaybe . drop 1 . T.words

-- | Car model value by make label and model label
modelField :: M.Map Text Text -> M.Map Text (M.Map Text Text) -> String -> String -> Field Text
modelField makers models fmake fmodel = do
    makeValue <- tableField makers fmake
    let
        modelTable = fromMaybe M.empty $ M.lookup makeValue models
    tableField modelTable fmodel

-- Models

ford :: Dict Model
ford = do
    fords <- getCarModels "ford"
    return $ model "ford" [
        car_buyDate <~ tryField "FIRST_REGISTRATION_DATE",
        car_checkupDate <~ tryField "VALID_FROM",
        car_checkupMileage <~ tryField "MILEAGE",
        car_make <~ pure "ford",
        car_model <~ tableField fords "MODEL",
        car_seller <~ tryField "DEALER_NAME",
        car_vin <~ field "VIN_NUMBER"]

fordPlus :: Dict Model
fordPlus = do
    fords <- getCarModels "ford"
    return $ model "fordPlus" [
        car_buyDate <~ tryField "Дата первой продажи",
        car_checkupDate <~ tryField "Дата прохождения ТО",
        car_checkupMileage <~ tryField "Пробег на момент прохождения ТО",
        car_make <~ pure "ford",
        car_model <~ tableField fords "Модель",
        car_vin <~ field "VIN"]

vwMotor :: Dict Model
vwMotor = do
    vws <- getCarModels "vw"
    cols <- asks colorsDict
    return $ model "vwMotor" [
        car_make <~ pure "vw",
        car_model <~ with (firstWord . tryField) "Модель" (look vws),
        car_color <~ with (takeColor . tryField) "Цвет авт" (lookKey cols),
        -- car_motor <~ with tryField "Модель" takeMotor,
        car_transmission <~ with_ tryField "Модель" takeTransmission,
        car_makeYear <~ tryField "Модельный год",
        car_vin <~ field "VIN",
        car_seller <~ tryField "Дилер получатель",
        car_buyDate <~ tryField "Дата договора продажи",
        ownerName <~ tryField "Контактное лицо покупателя"]
    where
        takeColor :: Field Text -> Field Text
        takeColor fld = fld >>= maybe empty return . fmap camelCase . takeColor' where
            takeColor' s = case fst . T.breakOnEnd "`" . snd . T.breakOn "`" $ s of
                res
                    | T.length res < 2 -> Nothing
                    | otherwise -> Just $ T.init $ T.tail res
            camelCase :: T.Text -> T.Text
            camelCase color = case T.words color of
                [] -> T.empty
                (c : cs) -> T.concat (T.map toLower c : map upperCase cs)
            upperCase :: T.Text -> T.Text
            upperCase = maybe T.empty (\(c, cs) -> T.cons (toUpper c) (T.map toLower cs)) . T.uncons

        takeMotor :: Text -> Either String Text
        takeMotor str = maybe err return . find isMotor . T.words $ str where
            isMotor s = T.length s == 3 && (T.index s 1 `elem` ".,")
            err = Left $ "Невозможно определить car_motor из '" ++ T.unpack str ++ "'"

        takeTransmission :: Text -> Maybe Text
        takeTransmission str = listToMaybe . mapMaybe fromTransmission . T.words $ str where
            fromTransmission s
                | "авт.-" `T.isPrefixOf` s = Just "auto"
                | "авт,-" `T.isPrefixOf` s = Just "auto"
                | "ручн.-" `T.isPrefixOf` s = Just "mech"
                | "ручн,-" `T.isPrefixOf` s = Just "mech"
                | "-мест" `T.isInfixOf` s = Just "mech"
                | otherwise = Nothing

        -- | Check whether key exists
        lookKey :: M.Map Text Text -> Text -> Either String Text
        lookKey tbl key = if key `elem` M.elems tbl then return key else err where
            err = Left $ "Неверное значение: '" ++ T.unpack key ++ "'"

vwCommercial :: Dict Model
vwCommercial = do
    vws <- getCarModels "vw"
    return $ model "vwcargo" [
        car_buyDate <~ tryField "Дата продажи",
        car_make <~ pure "vw",
        car_makeYear <~ tryField "модельный год",
        car_model <~ with (firstWord . tryField) "модель" (look vws),
        car_plateNum <~ tryField "госномер",
        car_seller <~ tryField "Продавец",
        car_vin <~ field "VIN",
        cardNumber <~ tryField "№ карты",
        ownerName <~ wordsFrom ["имя", "фамилия", "отчество"],
        ownerPhone <~ wordsFrom ["тел1", "тел2"],
        validUntil <~ tryField "Дата окончания карты"]

opel :: Dict Model
opel = do
    opels <- getCarModels "opel"
    return $ model "opel" [
        car_buyDate <~ tryField "Retail Date",
        car_make <~ pure "opel",
        car_model <~ tableField opels "Model",
        car_seller <~ tryField "Retail Dealer",
        car_vin <~ (field "VIN" <|> field "Previous VIN (SKD)")]

hummer :: Dict Model
hummer = do
    hums <- getCarModels "hum"
    return $ model "hum" [
        car_buyDate <~ tryField "Retail Date",
        car_make <~ pure "hum",
        car_model <~ with (secondWord . tryField) "Model" (look hums),
        car_seller <~ tryField "Retail Dealer",
        car_vin <~ field "VIN RUS"]

chevroletNAO :: Dict Model
chevroletNAO = do
    chevys <- getCarModels "chevy"
    return $ model "chevyna" [
        car_buyDate <~ tryField "Retail Date",
        car_make <~ pure "chevy",
        car_model <~ tableField chevys "Model",
        car_seller <~ tryField "Retail Dealer",
        car_vin <~ field "VIN RUS"]

chevroletKorea :: Dict Model
chevroletKorea = do
    chevys <- getCarModels "chevy"
    return $ model "chevyko" [
        car_buyDate <~ tryField "Retail Date",
        car_make <~ pure "chevy",
        car_model <~ with (firstWord . tryField) "Model" (look chevys),
        car_seller <~ tryField "Retail Dealer",
        car_vin <~ field "VIN"]

cadillac :: Dict Model
cadillac = do
    cads <- getCarModels "cad"
    return $ model "cadold" [
        car_buyDate <~ field "Retail Date",
        car_make <~ pure "cad",
        car_model <~ with (secondWord . field) "Model" (look cads),
        car_seller <~ field "Retail Dealer",
        car_vin <~ field "VIN RUS",
        cardNumber <~ field "Номер карты"]

vwRuslan :: Dict Model
vwRuslan = do
    vws <- getCarModels "vw"
    return $ model "ruslan" [
        car_make <~ pure "vw",
        car_model <~ with tryField "Модель Автомобиля VW" (look vws),
        car_vin <~ field "VIN номер Автомобиля VW",
        cardNumber <~ tryField "№",
        manager <~ tryField "ФИО ответственного лица, внесшего данные в XLS файл",
        milageTO <~ tryField "Величина пробега на момент регистрации в Программе",
        validFrom <~ tryField "Дата прохождения ТО (Дата регистрации в программе)",
        validUntil <~ tryField "Программа действует до (Дата)",
        validUntilMilage <~ tryField "Программа действует до (Пробега)"]

chartis :: Dict Model
chartis = do
    (makeFld, modelFld) <- makeModelParser
    return $ model "chartis" [
        car_make <~ makeFld "Марка Автомобиля",
        car_model <~ modelFld "Марка Автомобиля" "Модель Автомобиля",
        car_vin <~ field "VIN номер Автомобиля",
        cardNumber <~ tryField "Подрядковый номер клубной карты",
        manager <~ tryField "ФИО ответственного лица, внесшего данные в XLS файл",
        validFrom <~ tryField "Дата регистрации в программе",
        validUntil <~ tryField "Программа действует до (Дата)"]

vwAvilon :: Dict Model
vwAvilon = do
    vws <- getCarModels "vw"
    return $ model "avilon" [
        car_make <~ pure "vw",
        car_model <~ tableField vws "Модель Автомобиля VW",
        car_vin <~ field "VIN номер Автомобиля VW",
        cardNumber <~ tryField "Подрядковый номер клубной карты",
        manager <~ tryField "ФИО ответственного лица, внесшего данные в XLS файл",
        milageTO <~ tryField "Величина пробега на момент регистрации в Программе",
        serviceInterval <~ tryField "Межсервисный интервал",
        validFrom <~ tryField "Дата регистрации в программе",
        validUntil <~ tryField "Программа действует до (Дата)"]

atlantM :: Dict Model
atlantM = do
    vws <- getCarModels "vw"
    return $ model "atlant" [
        car_make <~ pure "vw",
        car_model <~ tableField vws "Модель Автомобиля VW",
        car_vin <~ field "VIN номер Автомобиля VW",
        cardNumber <~ tryField "Номер карты Atlant-M Assistance",
        manager <~ tryField "ФИО ответственного лица, внесшего данные в XLS файл",
        milageTO <~ tryField "Величина пробега на момент регистрации в Программе, км",
        -- program <~ field "Тип программы", -- ???
        serviceInterval <~ tryField "Межсервисный интервал, км",
        validFrom <~ tryField "Дата регистрации в программе",
        validUntil <~ tryField "Программа действует до (Дата)"]

autocraft :: Dict Model
autocraft = do
    bmws <- getCarModels "bmw"
    return $ model "autocraft" [
        car_make <~ pure "bmw",
        car_model <~ tableField bmws "Модель Автомобиля BMW",
        car_vin <~ field "VIN номер Автомобиля BMW (последние 7 знаков)",
        cardNumber <~ tryField "Подрядковый номер клубной карты",
        manager <~ tryField "ФИО ответственного лица, внесшего данные в XLS файл",
        milageTO <~ tryField "Величина пробега на момент регистрации в Программе",
        ownerName <~ tryField "ФИО Клиента",
        validFrom <~ tryField "Дата регистрации в программе",
        validUntil <~ tryField "Программа действует до (Даты)"]

europlan :: Dict Model
europlan = do
    (makeFld, modelFld) <- makeModelParser
    return $ model "euro" [
        car_make <~ makeFld "Марка автомобиля",
        car_makeYear <~ tryField "Год выпуска автомобиля",
        car_model <~ modelFld "Марка автомобиля" "Модель автомобиля",
        car_vin <~ field "VIN номер автомобиля",
        cardNumber <~ tryField "Подрядковый номер клубной карты",
        manager <~ tryField "ФИО ответственного лица, внесшего данные в XLS файл",
        -- program <~ field "Тип программы", -- ???
        validFrom <~ tryField "Дата регистрации в программе",
        validUntil <~ tryField "Программа действует до (Дата)"]

b2c :: Dict Model
b2c = do
    (makeFld, modelFld) <- makeModelParser
    return $ model "b2c" [
        car_make <~ makeFld "Марка автомобиля",
        car_makeYear <~ tryField "Год выпуска",
        car_model <~ modelFld "Марка автомобиля" "Модель автомобиля",
        car_plateNum <~ tryField "Гос номер",
        car_vin <~ field "Идентификационный номер (VIN)",
        cardNumber <~ tryField "Номер карты",
        manager <~ tryField "Сотрудник РАМК",
        ownerEmail <~ tryField "Е-МAIL клиента",
        ownerName <~ wordsFrom ["Ф<NAME>", "Имя клиента", "Отчество клиента"],
        ownerPhone <~ wordsFrom ["Телефон клиента Мобильный", "Телефон клиента Домашний"],
        -- program <~ tryField "Тип карты", -- ???
        validFrom <~ tryField "Дата активации карты",
        validUntil <~ tryField "Дата окончания срока дейсвия карты"]

citroenPeugeot :: String -> Dict Model
citroenPeugeot progname = do
    (makeFld, modelFld) <- makeModelParser
    return $ model progname [
        car_buyDate <~ tryField "FIRST_REGISTRATION_DATE",
        car_checkupMileage <~ tryField "MILEAGE",
        car_make <~ makeFld "MAKE",
        car_makeYear <~ tryField "VEHICLE_TYPE",
        car_model <~ modelFld "MAKE" "MODEL",
        car_plateNum <~ tryField "LICENCE_PLATE_NO",
        car_vin <~ field "VIN_NUMBER",
        car_warrantyEnd <~ tryField "VALID_TO",
        car_warrantyStart <~ tryField "VALID_FROM"]

-- | A common set for field mappings utilized by universal format.
universalFieldsCommon :: Dict [(String, Field Text)]
universalFieldsCommon = do
  (makeFld, modelFld) <- makeModelParser
  cols <- asks colorsDict
  return $
    [ car_buyDate <~ tryField "Дата покупки"
    , car_checkupDate <~ tryField "Дата последнего ТО"
    , car_color <~ tableField cols "Цвет"
    , car_make <~ makeFld "Марка"
    , car_makeYear <~ tryField "Год производства автомобиля"
    , car_model <~ modelFld "Марка" "Модель"
    , car_plateNum <~ tryField "Госномер"
    , cardNumber <~ tryField "Номер карты участника"
    , cardOwner <~ tryField "ФИО владельца карты"
    , manager <~ tryField "ФИО менеджера"
    , milageTO <~ tryField "Пробег при регистрации в программе"
    , ownerEmail <~ tryField "Email владельца"
    , ownerName <~ tryField "<NAME>"
    , ownerPhone <~ tryField "Контактный телефон владельца"
    , serviceInterval <~ tryField "Межсервисный интервал"
    , validFrom <~ tryField "Дата регистрации в программе"
    , validUntil <~ tryField "Программа действует до (дата)"
    , validUntilMilage <~ tryField "Программа действует до (пробег)"
    ]

universal :: Dict (String -> Model)
universal = do
    com <- universalFieldsCommon
    return $ \p -> model p $
                   com ++
                   [ car_vin <~ field "VIN"
                   ]

vtb24 :: Dict Model
vtb24 = model "vtb24" <$> universalFieldsCommon
<file_sep>/src/Vin/Utils.hs
module Vin.Utils
    ( DataRow
    , decodeICU
    )

where

import Data.ByteString (ByteString)

import qualified Data.Map as M

import Data.Text.ICU.Convert
import Data.Text.Encoding as T

type DataRow = M.Map ByteString ByteString

-- | Decode a row to UTF-8 using an ICU input converter.
decodeICU :: Converter -> DataRow -> DataRow
decodeICU c m = M.map enc m'
  where
    m' = M.mapKeys enc m
    enc =  T.encodeUtf8 . toUnicode c
<file_sep>/src/Vin/Model.hs
-- | Module with model definition and parse function
module Vin.Model (
    -- * Model description
    Model(..),
    model,
    parse
    ) where

import Control.Arrow ((***))
import Control.Applicative (pure)
import Data.Either (partitionEithers)
import Data.List (nub)
import qualified Data.Map as M
import Data.String (fromString)
import Data.Text (Text)

import Vin.Field

-- | Model description
data Model = Model {
    modelProgram :: String,
    modelMapping :: [(String, Field Text)] }

-- | Create @Model@
model :: String -> [(String, Field Text)] -> Model
model name maps = Model name $ (("program", text) <~ pure (fromString name)) : maps

-- | Parse one row, returns list of errors and parsed result
parse :: Model -> Row -> ([FieldError], Row)
parse m r = ((nub . concat) *** M.fromList) $ partitionEithers $ map parseField $ modelMapping m where
    parseField :: (String, Field Text) -> Either [FieldError] (Text, Text)
    parseField (name, parser) = do
        v <- evalField parser r
        case v of
            Nothing -> Left []
            Just v' -> return (fromString name, v')
<file_sep>/src/Vin/Store.hs
{-# LANGUAGE OverloadedStrings, DeriveDataTypeable #-}

module Vin.Store (
    VinUploadException(..),
    dbCreateVin, sinkXFile
    ) where

import Control.Arrow
import Control.Exception
import Control.Monad.IO.Class
import Control.Monad.Morph
import Control.Monad.Trans.Resource
import Control.Concurrent.MVar

import Data.ByteString (ByteString)
import qualified Data.ByteString.Char8 as C8

import Data.Conduit
import Data.Conduit.Binary
import qualified Data.Conduit.List as CL
import Data.CSV.Conduit hiding (Row, MapRow)

import Data.Char (isSpace)
import qualified Data.HashMap.Strict as HM
import qualified Data.Map as M
import qualified Data.Text as T
import qualified Data.Text.Encoding as T (encodeUtf8)
import Data.Typeable

import qualified System.IO as IO

import Carma.HTTP

import Vin.Field
import Vin.Model
import Vin.Load
import Vin.Utils

data VinUploadException = VinUploadException {
    vReason :: T.Text,
    vFilePath :: Maybe FilePath }
        deriving (Show, Typeable)

instance Exception VinUploadException

dbCreateVin :: ByteString
            -> ByteString
            -> Row
            -> ResourceT CarmaIO ()
dbCreateVin owner programRef val = do
  lift $ createInstance "contract" $
         HM.insert "owner" owner $
         HM.insert "program" programRef $
         HM.insert "dixi" "1" $
         HM.fromList $
         map (T.encodeUtf8 *** T.encodeUtf8) $
         M.toList val
  return ()

parseRow :: Model -> DataRow -> (Maybe String, Row)
parseRow m r = case parse m $ decodeRow r of
    ([], s) -> (Nothing, s)
    (es, s) -> (Just $ showErrors es, s)

trimKeys :: DataRow -> DataRow
trimKeys = M.mapKeys trim where
    trim = fst . C8.spanEnd isSpace . snd . C8.span isSpace

sinkXFile
    :: MonadResource m
    => (Row -> m ())
    -> FilePath
    -> FilePath
    -> (Int -> Int -> m ())
    -> Model
    -> Sink DataRowError m ()
sinkXFile store fError fLog stats ml
    =   safeMap trimKeys
    =$ safeMap (parseRow ml)
    =$ storeCorrect ml store stats
    =$ storeErrorLog fLog
    =$ writeIncorrect fError

storeCorrect
    :: MonadResource m
    => Model
    -> (Row -> m ())
    -> (Int -> Int -> m ())
    -> Conduit DataRowMaybeError m (DataRow, String)

storeCorrect _ store stats =
    let
        alloc = newMVar (0, 0)
        cleanup = const $ return ()
        updateStats m = liftIO (readMVar m) >>= uncurry stats
        newUpload m = liftIO (modifyMVar_ m (return . (succ *** succ))) >> updateStats m
        newFail m = liftIO (modifyMVar_ m (return . first succ)) >> updateStats m
    in
      bracketP alloc cleanup $
        \statsVar -> CL.mapMaybeM $
          \(src, r) -> case r of
            -- No errors, save store row
            Right (Nothing, dat) -> do
                store dat
                newUpload statsVar
                return Nothing
            -- Errors, store parsed row and produce errors
            Right (Just err, dat) -> do
                store dat
                newFail statsVar
                return $ Just (src, err)
            -- No parsed data, only error
            Left e' -> do
                newFail statsVar
                return $ Just (src, e')

storeErrorLog
    :: MonadResource m
    => FilePath
    -> Conduit (DataRow, String) m DataRow
storeErrorLog fLog = bracketP
    (do
        hFile <- IO.openFile fLog IO.WriteMode
        IO.hSetEncoding hFile IO.utf8
        return hFile)
    IO.hClose
    (\h -> CL.mapM (\(r, err) -> do
                      liftIO $ IO.hPutStrLn h err
                      return r))

writeIncorrect :: MonadResource m => FilePath -> Sink DataRow m ()
writeIncorrect fp = do
    res <- CL.peek
    case res of
      Nothing -> return ()
      Just _  -> do
          fp' <- writeRows fp
          throw $ VinUploadException "Errors during load: " (Just fp')

writeRows :: MonadResource m => FilePath -> Sink DataRow m FilePath
writeRows fp
    =  (writeHeaders csvSettings >> fromCSV csvSettings)
    =$ sinkFile fp >> return fp
    where
        csvSettings = defCSVSettings { csvSep = ';' }
<file_sep>/src/Vin/Field.hs
{-# LANGUAGE TypeSynonymInstances, MultiParamTypeClasses, FlexibleInstances, FlexibleContexts #-}

module Vin.Field (
    -- * Reading fields
    Row,
    decodeRow,
    FieldError(..),
    showError, showErrors,
    Field(..),
    fieldRead,
    FromField(..),
    field, tryField, with, with_,
    evalField,

    -- * Verify and postprocess fields
    FieldType,
    text, time, timeAt, int, email, vin,

    -- * Converters
    posix, posixZoned, look,

    -- * Special field readers
    timeField,
    tableField,

    -- * Combine
    (<:~),
    ModelField,
    (<~)
    ) where

import Control.Applicative
import Control.Monad.Reader
import Control.Monad.Writer

import Data.ByteString (ByteString)
import Data.Char (isSpace, toLower)
import Data.List (intercalate)
import Data.String (fromString)
import Data.Text (Text)
import qualified Data.Text as T
import qualified Data.Text.Encoding as T
import qualified Data.Map as M
import Data.Time
import Data.Time.Clock.POSIX
import System.Locale
import Text.Email.Validate (isValid)
import Text.Read (readMaybe)

-- | Row is Map from field name to its value
type Row = M.Map Text Text

-- | Decode @ByteString@ to @Text@ and trim whitespaces in @Row@
decodeRow :: M.Map ByteString ByteString -> Row
decodeRow = M.mapKeys decode . M.map decode where
    decode :: ByteString -> Text
    decode = T.dropAround isSpace . T.decodeUtf8

-- | Field error is
-- * Field not found
-- * Error reading field with field name and error description
data FieldError = NoField String | FieldError String String
    deriving (Eq, Ord, Read, Show)

-- | Show @FieldError@
showError :: FieldError -> String
showError (NoField name) = "Отсутствует поле '" ++ name ++ "'"
showError (FieldError name what) = "Ошибка в поле '" ++ name ++ "': " ++ what

-- | Show list of errors
showErrors :: [FieldError] -> String
showErrors = intercalate ", " . map showError

-- | Field reader
-- It's Functor, Applicative, Alternative and Monad.
-- Prefer using Applicative if you want to collect field errors:
--
-- >(++) <$> field "one" <*> field "two"
--
-- Will lead to two NoField errors, while
--
-- >do { x <- field "one"; y <- field "two"; return (x ++ y) }
--
-- Will fail on first @field@ with only one error
--
newtype Field a = Field {
    runField :: ReaderT Row (Writer [FieldError]) (Maybe a) }

instance Functor Field where
    fmap f = Field . fmap (fmap f) . runField

instance Applicative Field where
    pure = Field . pure . pure
    f <*> x = Field $ do
        f' <- runField f
        x' <- runField x
        return $ f' <*> x'

instance Alternative Field where
    empty = Field $ return Nothing
    l <|> r = Field $ do
        l' <- runField l
        r' <- runField r
        return $ l' <|> r'

instance Monad Field where
    return = Field . return . Just
    x >>= f = Field $ do
        x' <- runField x
        maybe (return Nothing) (runField . f) x'

instance MonadReader Row Field where
    ask = Field $ fmap Just ask
    local f (Field act) = Field $ local f act
    reader f = Field $ fmap Just $ reader f

-- | Read field with @read@, trimming whitespaces
fieldRead :: Read a => String -> Text -> Either String a
fieldRead typeName s = maybe err Right $ readMaybe str where
    str = T.unpack s
    err = Left $ "Невозможно преобразовать '" ++ str ++ "' в " ++ typeName

-- | Fail with error
fieldError :: FieldError -> Field a
fieldError e = do
    Field $ lift $ fmap Just $ tell [e]
    empty

-- | Type, that can be read from field
class FromField a where
    fromField :: Text -> Either String a

instance FromField a => FromField (Maybe a) where
    fromField s
        | T.null s = Right Nothing
        | otherwise = fmap Just $ fromField s

-- | Read field from @Row@
field :: FromField a => String -> Field a
field name = do
    value <- asks (M.lookup (fromString name))
    maybe (fieldError (NoField name)) (either (fieldError . FieldError name) return . fromField) value

-- | Try read field ignoring @NoField@ error and treating empty field as no input
tryField :: FromField (Maybe a) => String -> Field a
tryField name = do
    value <- asks (M.lookup (fromString name))
    maybe empty (either (fieldError . FieldError name) (maybe empty return) . fromField) value

-- | Perform failable action on field
-- Accepts field reader (@field@ or @tryField@ or even custom ones, like @tableField@),
-- field name and function
-- The only need is to get field name to throw proper FieldError on fail
--
-- >with field "foo" (look tbl)
-- >with tryField "bar" verifySmth
--
with :: (String -> Field a) -> String -> (a -> Either String b) -> Field b
with fld name f  = fld name >>= either (fieldError . FieldError name) return . f

-- | Perform action on field, and if result is Nothing, result to empty field
with_ :: (String -> Field a) -> String -> (a -> Maybe b) -> Field b
with_ fld name f = fld name >>= maybe empty return . f

-- | Evaluate @Field@, collecting errors and result.
evalField :: Field a -> Row -> Either [FieldError] (Maybe a)
evalField act r = case runWriter $ runReaderT (runField act) r of
    (Just v, _) -> Right (Just v)
    (Nothing, []) -> Right Nothing
    (Nothing, errs) -> Left errs

-- | @FieldType@ used to convert field data to @Text@.
type FieldType a = a -> Text

instance FromField Text where
    fromField = Right

-- | Dublin Julian time used in xlsx
instance FromField POSIXTime where
    fromField = fmap toPOSIX . fieldRead "время" where
        toPOSIX :: Double -> POSIXTime
        toPOSIX = toPOSIX' . properFraction where
            toPOSIX' (d, tm) = utcTimeToPOSIXSeconds utct where
                day = ModifiedJulianDay $ d - 2 + 2415020 - 2400000
                utct = UTCTime day (fromInteger . floor $ tm * 60 * 60 * 24)

-- | Try to parse LocalTime
-- Use @posix@ or @posixZoned@ to convert it to POSIXTime
instance FromField LocalTime where
    fromField s = maybe err Right $ msum $ map parse fmts where
        str = T.unpack s
        err = Left $ "Невозможно преобразовать '" ++ str ++ "' в дату"
        parse :: String -> Maybe LocalTime
        parse fmt = parseTime defaultTimeLocale fmt str
        fmts :: [String]
        fmts = [
            "%d/%m/%Y",
            "%e/%m/%Y",
            "%e/%m/%Y %k:%M",
            "%d.%m.%Y",
            "%e %b %Y",
            "%e-%b-%Y",
            "%d-%b-%y",
            "%d-%m-%y",
            "%d-%m-%Y",
            "%b %Y",
            "%Y-%m-%dT%H:%M:%S",
            "%d.%m.%Y %H:%M:%S"]

-- | Parse POSIXTime or LocalTime
instance FromField Time where
    fromField s = fmap TimePOSIX (fromField s) <|> fmap TimeLocal (fromField s)

instance FromField Int where
    fromField = fieldRead "число" . T.takeWhile (/= '.')

instance FromField Email where
    fromField s
        | isValid (T.unpack s) = Right $ Email s
        | otherwise = Left $ "'" ++ (T.unpack s) ++ "' не соответствует формату e-mail"

instance FromField VIN where
    fromField s = case map convert $ T.unpack s of
            str
                | length str == 17 -> Right $ VIN $ T.pack str
                | otherwise -> Left $ "Неверный VIN номер '" ++ str ++ "', должен состоять из 17 символов"
        where
        convert :: Char -> Char
        convert ch = case ch of
            'А' -> 'A'
            'В' -> 'B'
            'Е' -> 'E'
            'З' -> '3'
            'К' -> 'K'
            'М' -> 'M'
            'О' -> 'O'
            'Р' -> 'P'
            'С' -> 'C'
            'Т' -> 'T'
            'У' -> 'Y'
            'Х' -> 'X'
            _ -> ch

-- | Text field type
text :: FieldType Text
text = id

-- | Time field
data Time = TimePOSIX POSIXTime | TimeLocal LocalTime

-- | Time field type at 'MSK'
time :: FieldType Time
time = timeAt "+0400"

-- | Time at time zone
timeAt :: String -> FieldType Time
timeAt tz tm = showTime $ case tm of
    TimePOSIX posixTm -> posixTm
    TimeLocal localTm -> posixZoned (read tz) localTm
    where
        showTime :: POSIXTime -> Text
        showTime = T.pack . show' . floor where
            show' :: Integer -> String
            show' = show

-- | Integer field
int :: FieldType Int
int = T.pack . show

-- | Email
newtype Email = Email { getEmail :: Text } deriving (Eq, Ord, Read, Show)

-- | Email field type
email :: FieldType Email
email = getEmail

newtype VIN = VIN { getVIN :: Text } deriving (Eq, Ord, Read, Show)

-- | VIN field
vin :: FieldType VIN
vin = getVIN

-- | Convert @LocalTime@ to @POSIXTime@, using 'MSK' time zone
posix :: LocalTime -> POSIXTime
posix = posixZoned (read "+0400")

-- | Convert @LocalTime@ to @POSIXTime@ at time zone specified
posixZoned :: TimeZone -> LocalTime -> POSIXTime
posixZoned tz = utcTimeToPOSIXSeconds . localTimeToUTC tz

-- | Look value in table
look :: M.Map Text Text -> Text -> Either String Text
look tbl value = maybe err Right $ M.lookup (lower value) tbl' where
    lower :: Text -> Text
    lower = T.map toLower
    tbl' :: M.Map Text Text
    tbl' = M.mapKeys lower tbl
    err :: Either String Text
    err = Left $ "Неверное значение: '" ++ T.unpack value ++ "'"

-- | Tries to read field as POSIXTime and LocalTime at 'MSK'
timeField :: String -> Field POSIXTime
timeField name = field name <|> (fmap posix $ field name)

-- | Lookup field value in @Map@
tableField :: M.Map Text Text -> String -> Field Text
tableField tbl name = with tryField name $ look tbl

-- | Combine @Field@ and @FieldType@, making function, which will read field,
-- producing possibly empty output or error.
(<:~) :: FieldType a -> Field a -> Field Text
ftype <:~ fld = fmap ftype fld

-- | Field name with @FieldType@
type ModelField a = (String, FieldType a)

-- | Combine @ModelField@ and @Field@
(<~) :: ModelField a -> Field a -> (String, Field Text)
(name, ftype) <~ fld = (name, ftype <:~ fld)
<file_sep>/src/Vin/Import.hs
module Vin.Import (
    VinUploadException(..),
    ContentType,
    contentType, extension,
    importData,
    loadFile
    ) where

import Control.Exception (throw)
import Control.Monad.IO.Class

import Data.ByteString (ByteString)
import Data.Conduit
import qualified Data.Map as M

import qualified Data.Text as T (pack)

import Carma.HTTP

import Vin.Model
import Vin.Models (models, runDict)
import Vin.Load
import Vin.Store

-- | Try to get maybe value
try :: Maybe a -> String -> IO a
try Nothing msg = throw $ VinUploadException (T.pack msg) Nothing
try (Just x) _ = return x

-- | Content type
type ContentType = Either String String

-- | By ContentType
contentType :: String -> ContentType
contentType = Left

-- | By file extension
extension :: String -> ContentType
extension = Right

-- | Import data
-- TODO: Move to import
importData
    :: (String -> Maybe Model)
    -- ^ Lookup program by its name.
    -> FilePath
    -- ^ Input file
    -> FilePath
    -- ^ Errors file
    -> FilePath
    -- ^ Log file
    -> ByteString
    -- ^ Owner field value
    -> ByteString
    -- ^ Program name
    -> String
    -- ^ VIN file format name.
    -> ContentType
    -- ^ Content type
    -> (Int -> Int -> ResourceT CarmaIO ())
    -- ^ Progress
    -> Int
    -- ^ CaRMa port.
    -> IO ()

importData ms from failed errors owner program vinFormat content stats cp = do
    loader <- liftIO $ try (either (`M.lookup` loadersContentType) (`M.lookup` loadersExtension) content) "Unknown loader"
    -- loader <- try (M.lookup content ls) $ "Unknown loader"
    m <- liftIO $ try (ms vinFormat) $ "Unknown program"
    l <- liftIO $ loader from
    runCarma defaultCarmaOptions{carmaPort = cp} $ runResourceT $ (l $$ sinkXFile (dbCreateVin owner program) failed errors stats m)

loadFile :: FilePath -> FilePath -> FilePath -> ByteString -> ByteString -> String -> ContentType -> (Int -> Int -> ResourceT CarmaIO ()) -> Int -> IO ()
loadFile iFile eFile lFile owner pName vinFormat cType stats cp = do
    models' <- runDict models
    models'' <- try models' $ "Unable to load models"
    importData models'' iFile eFile lFile owner pName vinFormat cType stats cp
<file_sep>/src/Vin/Text/Specific.hs
module Vin.Text.Specific (
    carModels,
    carMakers
    ) where

import Data.ByteString (ByteString)
import qualified Data.Map as M

import Vin.Text

(<<=~) :: String -> [String] -> M.Map ByteString ByteString
str <<=~ alts = str <<~ (str : alts)

-- | Car models
carModels :: FieldType ByteString
carModels = table $ M.unions [
  -- VW truck
  "Caddy"      <<=~ ["Кэдди", "кедди", "Кедди"],
  "Amarok"     <<=~ [],
  "Crafter"    <<=~ ["Крафтер"],
  "Transporter"<<=~ ["T5", "Т5", "Транспортер"],
  -- VW motor
  "Tiguan"     <<=~ ["Тигуан", "тигуан"],
  "Polo"       <<=~ ["Поло"],
  "Touareg"    <<=~ ["Туарег", "Тouareg"],
  "Passat"     <<=~ ["Пассат", "пассат", "Passft"],
  "Jetta"      <<=~ ["Джетта"],
  "Golf"       <<=~ ["Гольф", "гольф", "Гольф+"],
  "Touran"     <<=~ ["Туран"],
  "Phaeton"    <<=~ ["Фаэтон", "фаэтон"],
  "Eos"        <<=~ ["Эос"],
  "Scirocco"   <<=~ ["Сирокко"],
  "Caravelle"  <<=~ ["Каравелла"],
  "Multivan"   <<=~ ["Мультивен"],
  "Sharan"     <<=~ ["Шаран"],
  -- Opel
  "Astra"      <<=~ [],
  "Zafira"     <<=~ [],
  "Corsa"      <<=~ [],
  "Insignia"   <<=~ [],
  "Combo"      <<=~ [],
  "Meriva"     <<=~ [],
  "Antara"     <<=~ [],
  "Vectra"     <<=~ [],
  -- Hummer
  "FOCUS"      <<=~ ["Фокус", "Focus"],
  "ESCAPE"     <<=~ [],
  "MONDEO"     <<=~ ["Мондео", "Mondeo"],
  "FIESTA"     <<=~ ["Fiesta"],
  "FUSION"     <<=~ ["Fusion"],
  "COUGAR"     <<=~ [],
  "KUGA"       <<=~ [],
  "GALAXY"     <<=~ [],
  "EXPLORER"   <<=~ [],
  "MAVERICK"   <<=~ [],
  "TRANSIT"    <<=~ ["Transit"]]

-- | Car makers
carMakers :: FieldType ByteString
carMakers = byteString
<file_sep>/src/Snap/Snaplet/Vin.hs
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE TemplateHaskell #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE DeriveGeneric #-}

module Snap.Snaplet.Vin
    ( Vin
    , vinInit
    , initUploadState, uploadData, getState, removeAlert
    ) where

import           Control.Concurrent
import qualified Control.Exception as E
import           Control.Monad.IO.Class
import           Control.Monad.Trans.Resource
import           Data.ByteString (ByteString)
import           Data.Map as M
import           Data.Maybe
import           GHC.Generics

import           Control.Lens
import           Control.Monad.State
import           Data.Aeson
import           Data.String
import           Data.Text (Text)
import qualified Data.Text as T
import qualified Data.Text.Encoding as T
import           Snap.Core
import           Snap.Http.Server.Config as S
import           Snap.Snaplet
import           System.FilePath

import           Carma.HTTP (CarmaIO)

import           Vin.Import

data Alert a = Alert {
    alertId :: a,
    alertType :: Text,
    alertMessage :: Text,
    alertVinFile :: Text,
    alertErrorFile :: Maybe FilePath,
    alertErrorLogFile :: Maybe FilePath  }
        deriving (Eq, Ord, Read, Show, Generic)

instance ToJSON a => ToJSON (Alert a)

newtype Alerts a = Alerts (Map a (Alert a))

instance ToJSON a => ToJSON (Alerts a) where
    toJSON (Alerts m) = object [("alerts", alerts')] where
        alerts' = toJSON $ M.elems m

insertAlert :: Ord a => Alert a -> Alerts a -> Alerts a
insertAlert a (Alerts m) = Alerts $ M.insert (alertId a) a m

updateAlert :: Ord a => Alert a -> Alerts a -> Alerts a
updateAlert a (Alerts m) = Alerts $ M.update (const $ Just a) (alertId a) m

deleteAlert :: Ord a => a -> Alerts a -> Alerts a
deleteAlert k (Alerts m) = Alerts $ M.delete k m

------------------------------------------------------------------------------
data Vin = Vin {
    _alerts :: MVar (Alerts Text) }

makeLenses ''Vin

withAlerts :: MVar a -> (a -> a) -> IO ()
withAlerts m f = modifyMVar_ m (return . f)

alertInsert :: Ord a => MVar (Alerts a) -> Alert a -> IO ()
alertInsert mvar a = withAlerts mvar (insertAlert a)

alertUpdate :: Ord a => MVar (Alerts a) -> Alert a -> IO ()
alertUpdate mvar a = withAlerts mvar (updateAlert a)

alertDelete :: Ord a => MVar (Alerts a) -> a -> IO ()
alertDelete mvar k = withAlerts mvar (deleteAlert k)

-- | Generic alert
alert :: Text -> a -> Text -> Text -> Alert a
alert t i msg vinFile = Alert i t msg vinFile Nothing Nothing

-- | Info alert
infoAlert :: a -> Text -> Text -> Alert a
infoAlert = alert "info"

-- | Success alert
successAlert :: a -> Text -> Text -> Alert a
successAlert = alert "success"

-- | Error alert
errorAlert :: a -> Text -> Text -> Alert a
errorAlert = alert "error"

-- | Add error file to alert
withErrorFile :: Alert a -> FilePath -> Alert a
withErrorFile a f = a { alertErrorFile = Just f }

-- | Add error log file to alert
withErrorLogFile :: Alert a -> FilePath -> Alert a
withErrorLogFile a f = a { alertErrorLogFile = Just f }

-- | Obtain port number of the Snap application.
currentPort :: IO Int
currentPort = do
  sCfg <- commandLineConfig (emptyConfig :: S.Config Snap a)
  return $ case getPort sCfg of
             Just n -> n
             Nothing -> error "No port"

initUploadState :: String -> Handler b Vin ()
initUploadState f = do
    s <- gets _alerts
    liftIO $ alertInsert s $ infoAlert (T.pack f) "Uploading..." (T.pack f)

uploadData :: ByteString
           -- ^ Owner field value.
           -> ByteString
           -- ^ Parent program reference (@program:12@).
           -> String
           -- ^ Vin format name.
           -> FilePath
           -- ^ Path to a VIN data file.
           -> Handler b Vin ()
uploadData owner program vinFormat fUploaded = do
    s <- gets _alerts
    statsVar <- liftIO $ newMVar (0, 0)

    let
        uploadStats :: Int -> Int -> ResourceT CarmaIO ()
        uploadStats total valid = liftIO $ do
            swapMVar statsVar (total, valid)
            alertUpdate s $ infoAlert (T.pack f) msg (T.pack f)
            where
                msg = T.concat [
                    "Uploading... total rows processed: ",
                    T.pack $ show total,
                    ", rows uploaded: ",
                    T.pack $ show valid]

        endWith :: Maybe T.Text -> IO ()
        endWith result = do
            (total, valid) <- readMVar statsVar
            let
                statsMsg = T.concat [
                    "Total rows processed: ",
                    T.pack $ show total,
                    ", rows uploaded: ",
                    T.pack $ show valid]
                resultMsg = T.concat [
                    maybe "Done. " (\err -> T.concat ["Failed with: ", err, ". "]) result,
                    statsMsg]
            alertUpdate s $ (if isJust result then errorAlert else successAlert) (T.pack f) resultMsg (T.pack f)
                `withErrorFile` fErrorLink
                `withErrorLogFile` fLogLink

    carmaPort <- liftIO currentPort
    liftIO $ forkIO $
        (loadFile fUploaded fError fLog owner program vinFormat (extension $ takeExtension fUploaded) uploadStats carmaPort >> endWith Nothing)
        `E.catches` [
            E.Handler (\(ex :: VinUploadException) -> endWith Nothing),
            E.Handler (\(ex :: E.SomeException) -> endWith (Just $ fromString $ show ex))]

    writeBS "Ok"
    where
        unquote "" = ""
        unquote s
            | head s == '"' = init . tail $ s
            | otherwise = s
        f = takeFileName fUploaded
        partFs = unquote f
        fError = "resources" </> "static" </> (partFs ++ ".error.csv")
        fLog = "resources" </> "static" </> (partFs ++ ".error.log")
        fErrorLink = "s/" ++ partFs ++ ".error.csv"
        fLogLink = "s/" ++ partFs ++ ".error.log"

getState :: Handler b Vin ()
getState = do
    s <- gets _alerts
    alerts' <- liftIO $ readMVar s
    writeLBS $ encode alerts'

removeAlert :: Handler b Vin ()
removeAlert = do
    s <- gets _alerts
    res <- getParam "id"
    maybe (return ()) (liftIO . alertDelete s . T.decodeUtf8) res

routes :: [(ByteString, Handler b Vin ())]
routes = [
    ("/state", method GET getState),
    ("/state", method POST removeAlert)]

vinInit :: SnapletInit b Vin
vinInit = makeSnaplet "vin" "Some description" Nothing $ do
    alerts' <- liftIO . newMVar $ Alerts M.empty
    -- addRoutes routes
    return $ Vin alerts'
<file_sep>/src/Vin/Models/Cars.hs
{-# LANGUAGE OverloadedStrings #-}

module Vin.Models.Cars (
    loadValue,
    getObject, getMember,
    Entry(..)
    ) where

import Control.Applicative
import Data.Aeson
import qualified Data.ByteString.Lazy.Char8 as C8
import qualified Data.HashMap.Strict as HM
import qualified Data.Text as T

-- | Load JSON value from file
loadValue :: FilePath -> IO (Maybe Value)
loadValue f = fmap decode $ C8.readFile f

-- | Get member object
getObject :: Value -> [String] -> Maybe Value
getObject v [] = Just v
getObject (Object obj) (s:ss) = HM.lookup (T.pack s) obj >>= (`getObject` ss)
getObject _ _ = Nothing

-- | Get member value (parsed object)
getMember :: FromJSON a => Value -> [String] -> Maybe a
getMember v m = do
    v' <- getObject v m
    case fromJSON v' of
        Success x -> Just x
        Error _ -> Nothing

-- | Car label
data Entry = Entry {
    value :: T.Text,
    label :: T.Text }
        deriving (Eq, Ord, Read, Show)

instance FromJSON Entry where
    parseJSON (Object v) = Entry <$> (v .: "value") <*> (v .: "label")
    parseJSON _ = empty
<file_sep>/src/Vin/Load.hs
module Vin.Load (
    DataError,
    DataRowError, DataRowMaybeError,
    csv, xlsx,
    Loader, loadersContentType, loadersExtension,
    dup, safeMap, safeMapM
    ) where

import Control.Exception
import Control.DeepSeq
import Control.Monad.IO.Class (liftIO)
import Data.Conduit
import Data.Conduit.Binary
import qualified Data.Conduit.List as CL
import qualified Data.Map as M
import Data.CSV.Conduit hiding (Row, MapRow)

import qualified Codec.Xlsx.Parser as Xlsx

import qualified Data.Text.Encoding as T
import Data.Text.ICU.Convert as ICU

import Vin.Field (Row)
import Vin.Utils

type DataError c a = (c, Either String a)

type DataRowError = DataError DataRow DataRow

-- | Parsed DataRow with parse errors
type DataRowMaybeError = DataError DataRow (Maybe String, Row)

-- | Load CSV
csv
    :: MonadResource m
    => FilePath
    -> Source m DataRowError
csv f = do
  let csvSettings = defCSVSettings { csvSep = ';' }
  conv <- liftIO $ ICU.open "cp1251" Nothing
  sourceFile f $= intoCSV csvSettings $= CL.map dup $= safeMap (decodeICU conv) where

-- | Load XLSX
xlsx
    :: MonadResource m
    => FilePath
    -> IO (Source m DataRowError)
xlsx f = do
    x <- Xlsx.xlsx f
    return (Xlsx.sheetSource x 0 Xlsx.convertToText Xlsx.convertToText $= CL.map (dup . encode))

-- FIXME: Trick with IO used for xlsx to preload file and then create source from it
-- I don't know the right way to do this
type Loader m = FilePath -> IO (Source m DataRowError)

-- | Loaders by content type
loadersContentType :: MonadResource m => M.Map String (Loader m)
loadersContentType = M.fromList [
    ("text/csv", return . csv),
    ("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", xlsx)]

loadersExtension :: MonadResource m => M.Map String (Loader m)
loadersExtension = M.fromList [
    (".csv", return . csv),
    (".xlsx", xlsx)]

-- Old functions:

encode :: Xlsx.MapRow -> DataRow
encode m = M.map T.encodeUtf8 m' where
        m' = M.mapKeys T.encodeUtf8 m

dup :: a -> DataError a a
dup r = (r, Right r)

-- | Wrap pure function within IO to catch exceptions and provide source data on error
safeMap :: (MonadResource m, NFData b) => (a -> b) -> Conduit (DataError c a) m (DataError c b)
safeMap f = safeMapM (Right . f)

safeMapM :: (MonadResource m, NFData b) => (a -> Either String b) -> Conduit (DataError c a) m (DataError c b)
safeMapM f = CL.mapM
    (\(r, i) -> case i of
        Left x -> return (r, Left x)
        Right v -> liftIO $ catch (produce f r v) (onError r))
    where
        produce :: NFData b => (a -> Either String b) -> c -> a -> IO (DataError c b)
        produce f' r v = do
            v' <- evaluate $ force $ f' v
            return (r, v')
        onError :: c -> SomeException -> IO (DataError c a)
        onError r e = return $ (r, Left $ show e)
<file_sep>/src/Vin/Text/String.hs
module Vin.Text.String (
    encodeString, decodeString, withString
    ) where

import Data.ByteString (ByteString)

import qualified Data.Text as T
import qualified Data.Text.Encoding as E

-- | Encode string to UTF8
encodeString :: String -> ByteString
encodeString = E.encodeUtf8 . T.pack

-- | Decode string from UTF8
decodeString :: ByteString -> String
decodeString = T.unpack . E.decodeUtf8

-- | Perform action on decoded string
withString :: (String -> String) -> ByteString -> ByteString
withString f = encodeString . f . decodeString
<file_sep>/src/Vin/Text/DateTime.hs
module Vin.Text.DateTime (
    timeFormat,
    posixLocale, posix,
    dublin
    ) where

import Control.Applicative
import Control.Monad.Error

import Data.ByteString (ByteString)
import qualified Data.ByteString.Char8 as C8
import Data.Char (isSpace)
import Data.Time
import Data.Time.Clock.POSIX
import System.Locale

import Vin.Text.String
import Vin.Field

-- | Date-time with format and locale
timeFormat :: (ParseTime t, Error e) => TimeLocale -> String -> Field e ByteString t
timeFormat loc fmt = do
    s <- field
    let
        cantParse = "Unable to parse time " ++ decodeString s ++ " with format " ++ fmt
    maybe (throwError $ strMsg cantParse) return $ parseTime loc fmt (decodeString s)

-- | Posix time with format
posixLocale :: Error e => TimeLocale -> String -> Field e ByteString POSIXTime
posixLocale loc fmt = utcTimeToPOSIXSeconds <$> timeFormat loc fmt

-- | Posix time with format and default locale
posix :: Error e => String -> Field e ByteString POSIXTime
posix = posixLocale defaultTimeLocale

-- | Dublin Julian date used it xlsx.
dublin :: Error e => Field e ByteString POSIXTime
dublin = field >>= toDublin where
    toDublin str = case tryDouble (trim str) of
        Nothing -> throwError $ strMsg $ "Unable to parse dublin time: " ++ decodeString str
        Just d -> return $ toPOSIX d

    trim = snd . C8.span isSpace . fst . C8.spanEnd isSpace
    tryDouble s = case reads (C8.unpack s) of
        [(v, "")] -> return v
        _ -> Nothing

    toPOSIX :: Double -> POSIXTime
    toPOSIX d = toPOSIX' $ properFraction d where
        toPOSIX' (d', tm) = utcTimeToPOSIXSeconds utct where
            day = ModifiedJulianDay $ d' - 2 + 2415020 - 2400000
            utct = UTCTime day (fromInteger . floor $ tm * 60 * 60 * 24)

languages:
Cabal Config
Haskell

repo_language: Cabal Config

repo_name: f-me/vin-parser


text:
<file_sep># Disaster Recovery Plan - Thinkofliving

- [Overview](#overview)
- [System Information](#system-information)
  - [Service Components](#service-components)
- [Prerequisite](#prerequisite)
  - [Access Requirements](#access-requirements)
  - [Infrastructure](#infrastructure)
- [Upstream Dependant Services](#upstream-dependant-services)
- [Downstream Dependant Services](#downstream-dependant-services)
- [Disaster Scenarios and Recovery Procedures](#disaster-scenarios-and-recovery-procedures)
  - [Scenario A: Availability Zone Outage](#1-scenario-a-aws-single-az-outage)
  - [Scenario B: Individual Service Outage](#2-scenario-b-aws-individual-service-outage)
  - [Scenario C: Region Outage](#3-scenario-c-aws-region-outage)
  - [Scenario D: Data Loss](#4-scenario-d-data-loss)
- [Notes](#notes)
- [Appendix](#appendix)
  - [TOL Auto Scaling Configurations](#tol-auto-scaling-configurations)
  - [TOL CDN Distribution Table](#tol-cdn-distribution-table)

## Overview
The purpose of this plan is to list out steps involved in recovering the system components in ***TOL*** application. The system is hosted in Amazon AWS in a high available (multi-AZ) environment in ap-southeast-1 (Singapore) region.

## System Information
Below are the essential information related to TOL application.
 - [Architecture Diagram](https://iproperty.pagerduty.com/escalation_policies#PA1RTXS)
 - [1Pager Document](https://iproperty.pagerduty.com/escalation_policies#PA1RTXS)

#### Service Components

**1. Web Frontend (Container)**
The front end web application accessed by the general public or wide user base. Developed on React and NextJS framework.

**2. API (Container)**
The GraphQL API application that acts as an interface between the Web Frontend and backend data source. The data source comes from Elastic Search (Primary) and Wordpress API (Fallback). Developed on Nodejs.

**3. CMS/Wordpress API (Container)**
The wordpress application used by internal editors to maintain content which is displayed by the web frontend. Also consists of API which interacts with wordpress database. Runs on PHP.

**4. Wordpress Database (RDS)**
The MySQL database for the Wordpress application. This is where data originates and is the source of truth for the TOL system. This database is accessed by only the Wordpress API, not directly by any other application.

**5. Elastic Search (Elasticsearch Service)**
The scalable and responsive data source which indexes the JSON output from the WP API. This data source acts as an offset to the Wordpress API as the Wordpress API is unable to handle high loads effectively.

## Prerequisite

#### Access Requirements
Below are the access required to run the Disater Recovery Procedures.

| System  | Role |
| --------| ---- |
| AWS     | Production: tol-production-Privileged (103426260373) |

#### Infrastructure

The following table provides an overview of infra services running in the backup region.

| Global | Singapore (Primary) | Tokyo (DR) | Remarks |
| --------| ----------- | ----- | ----- |
| Route53| | | thinkofliving.com |
| CloudFront| | |
| Lambda Edge| | |
| WAF | | |
| | EC2 (Bastion Host)| EC2 (Bastion Host)| |
| | Fargate| Fargate| |
| | S3| S3| Assets: https://cdn-assets.prod.thinkofliving.com/ |
| | RDS/Aurora| RDS/Aurora| arn:aws:rds:ap-southeast-2:855953295449:db:thinkofliving |
| | Elasticsearch Service | | |
| | VPC & Subnet | VPC & Subnet| |
| | CloudWatch| CloudWatch| |
| | ALB| ALB| |
| | Lambda & API Gateway | Lambda & API Gateway | |
| | KMS| KMS| |

## Upstream Dependant Services
TOL application is relies on the following upstream services.

| Service | Description | Owner |
| ------ | ------ | ------ |
| Sharpie | Images served on the site | Platform |
| Locke | User login and registration | User Management (UM) |

## Downstream Dependant Services
There are no downstream dependant services in TOL.

## Disaster Scenarios and Recovery Procedures
Below are the procedure and scenarios captured for a disaster.
### Communication:
In case of any of the below disaster scenarios, please follow [incident process](https://garage.rea-group.com/documents/172793/developer-consumer-incident-process) and execute the recovery procedure in following order:
1. Send an **@channel** notification on Slack at **#asia-content-delivery-squad** and **#tol-ongoing**
2. Communicate to all relevant stakeholders
3. Create bunker slack channel for communications for the incident and DR
4. Assemble a team and execute recovery procedure depending on the particular disaster scenario as below:

### Scenarios:
- Scenario A: AZ failure in Singapore (ap-southeast-1a or ap-southeast-1b)
  - All services can be automatically failover to other healthy AZ.
- Scenario B: Individual service failure in Singapore (all availability zones)
  - Depending on which service. If ElasticSearch or RDS, application operate with minor functionality loss.
- Scenario C: Region failure in Singapore
  - Start up all services in Tokyo (ap-northeast-1) and divert live traffic.
- Scenario D: Data Loss (RDS)
  - Restore MySQL dump file from S3.

### 1. Scenario A: AWS Single AZ Outage

#### 1.1 Impact
**Minor** - due to automatic failover capability provided by AWS and handled by Auto Scaling Group settings.

#### 1.2 Estimated Time for Recovery
None required

#### 1.3 Procedure
Automatic. No manual intervention required.

### 2. Scenario B: AWS Individual Service Outage
In the event that a particular service is unavailable in the Singapore region.

#### 2.1 Impact
- **Minor**:
  - **Elastic Search**: No action required. API fail safe will automatically fall back to RDS.
  - **RDS**: No action required. Application can remain operable with certain functionality disabled as long as Elastic Search remains operable.
  *Impaired functionality include: Comments, Free Text Search, Price Filter Search, Login, Register, User Profile*.
- **Severe**
  - **Other Service(s)**: Will need to be treated similar to Scenario C: AWS Region Outage.

#### 2.2 Estimated Time for Recovery
- **Elastic Search or RDS faillure**: no recovery time required.
- **Elastic Search and RDS faillure**: The recovery time is same as Scenario C.
- **Other service faillure**: The recovery time is same as Scenario C.

#### 2.3 Procedure
- **Elastic Search or RDS faillure**: Automatic. No manual intervention required.
- **Elastic Search and RDS faillure**: Please refer to Scenario C.
- **Other service faillure**: Please refer to Scenario C.

### 3. Scenario C: AWS Region Outage

#### 3.1 Impact
**Severe** - if both the Availability Zones (ap-southeast-1a and ap-southeast-1b) in Singapore region have an outage. All [Service Components](#service-components) have to be up and running in the backup Tokyo region in order to resume availability.

#### 3.2 Recovery Procedure
Estimated Time for Recovery: 30 minutes

**Activate AWS Tokyo Region and Divert Traffic**

  - 3.2.1. Upscale Elastic Search in backup region
  - 3.2.2. Upscale RDS in backup region
  - 3.2.3. Increase Fargate task count for Web Frontend, API, and CMS applications in backup region
  - 3.2.4. Update CloudFront origin to switch live traffic to backup region
  - 3.2.5. Invalidate CloudFront cache
  - 3.2.6. Verify that site is up and running

  **3.2.1. Upscale Elastic Search in backup region**

  a. Login to AWS Console in Production, then switch to Tokyo Region.
  ![RDS](assets/TOL-DR-AWS-RoleSelection.png?raw=true "RDS")

  b. Go to **Services > Elasticsearch Service**, then click on the domain `th-thin-elasti...`.
  ![RDS](assets/TOL-DR-ES.png?raw=true "RDS")

  c. On the top left, click on `Edit Domain`.
  ![RDS](assets/TOL-DR-ES-EditDomain.png?raw=true "RDS")

  d. Scroll down to **Data nodes** section and set the data nodes according to the configuration shown:
  ![RDS](assets/TOL-DR-ES-DataNodes.png?raw=true "RDS")

  e. Scroll down to **Dedicated master nodes** section and set the master nodes according to the configuration shown:
  ![RDS](assets/TOL-DR-ES-MasterNodes.png?raw=true "RDS")

  e. Scroll down to bottom and click **Submit** button.

  f. Complete. Move to the next step.

  **3.2.2. Upscale RDS in backup region**

  a. Login to AWS Console in Production, then switch to Tokyo Region.
  ![RDS](assets/TOL-DR-AWS-RoleSelection.png?raw=true "RDS")

  b. Go to **Services > Amazon RDS**. In Amazon RDS, click on **DB Instances**
  ![RDS](assets/TOL-DR-RDS.png?raw=true "RDS")

  b. Select the DB Identifier **th-thinkofliving-prod-rds** then click the Modify button
  ![RDSModify](assets/TOL-DR-RDS-Modify.png?raw=true "RDSModify")

  c. In Modify Database, scroll to DB Instance size, then select "Memory Optimized classes" and choose from the list **db.r5.large**. Scroll down and click the "Continue" button.
  ![RDSScaleUp](assets/TOL-DR-RDS-ScaleUp.png?raw=true "RDSScaleUp")

  d. Complete. Move to the next step.

  **3.2.3. Increase Fargate task count Web Frontend application**

  a. Login to AWS Console in Production, then switch to Tokyo Region.
  ![RDS](assets/TOL-DR-AWS-RoleSelection.png?raw=true "RDS")

  b. Go to **Services > Elastic Container Service**. In Clusters, click on 'ecs-cluster-tol-blue' cluster
  ![ECS Clusters](assets/TOL-DR-Clusters.png?raw=true "ECS Clusters")

  c. Under the Services tab, click on **tol-frontend-blue-EcsService...**
  ![ECS Services](assets/TOL-DR-ClusterServices.png?raw=true "ECS Services")

  d. Click the 'Update' button on the top right of services page
  ![Update Service](assets/TOL-DR-UpdateService.png?raw=true "Update Service")

  e. In Step 1 and 2, scroll to the bottom and click 'Next' until you reach the 'Step 3: Set Auto Scaling (optional) page. On this page, set the task counts according to the following table: [TOL Auto Scaling Configurations](#tol-auto-scaling-configurations). When done, scroll to the bottom and ckick 'Next'
  ![Update Scaling](assets/TOL-DR-UpdateServiceScaling.png?raw=true "Update Scaling")

  f. In Step 4, scroll to the bottom and click on 'Update Service'
  ![Confirm Update](assets/TOL-DR-UpdateServiceConfirm.png?raw=true "Confirm Update")

  g. Repeat steps (c) to (f) with the following ECS services: **tol-api-blue-EcsService...** and **tol-wordpress-blue-EcsService...**

  **3.2.4. Update CloudFront origin**

  a. Login to AWS Console in Production, then switch to Tokyo Region.
  ![RDS](assets/TOL-DR-AWS-RoleSelection.png?raw=true "RDS")

  b. Go to **Services > CloudFront** In CloudFront Distributions, select the checkbox on the same row with Comment **thinkofliving-wordpress**. Then click on `Distribution Settings`
  ![CloudFront](assets/TOL-DR-CF.png?raw=true "CloudFront")

  c. In Distribution Settings, go to the `Origins and Origin Groups` tab, select the checkbox next to the first row then click on `Edit`
  ![CloudFront Behaviors](assets/TOL-DR-CF-Origins.png?raw=true "CloudFront Behaviors")

  d. In Edit Origin, change the Origin Domain Name to **wordpress-active-tk.prod.thinkofliving.com**. Then scroll to the bottom and click on 'Yes, Edit' button.
  ![CloudFront Edit Behavior](assets/TOL-DR-CF-EditOrigins.png?raw=true "CloudFront Edit Behavior")

  e. Repeat step (b) to (d) for each distribution in the [TOL CDN Distribution Table](). Set the origin according to the DR (Tokyo) region origin.

  **3.2.5. Invalidate CloudFront cache**

  a. Login to AWS Console in Production, then switch to Tokyo Region.
  ![RDS](assets/TOL-DR-AWS-RoleSelection.png?raw=true "RDS")

  b. Go to **Services > CloudFront** In CloudFront Distributions, select the checkbox on the same row with Comment **thinkofliving-wordpress**. Then click on 'Distribution Settings'
  ![CloudFront](assets/TOL-DR-CF.png?raw=true "CloudFront")

  c. In Distribution Settings, go to the 'Invalidations' tab, then click on 'Create Invalidation'
  ![CloudFront Create Invalidation](assets/TOL-DR-CF-CreateInvalidation.png?raw=true "CloudFront Create Invalidation")

  d. Enter "*" (without quotes) into the Object Paths textbox. Then click on 'Invalidate'
  ![CloudFront Invalidation](assets/TOL-DR-CF-Invalidate.png?raw=true "CloudFront Invalidation")

  e. Repeat steps (b) to (d) with the following CloudFront Distributions: **thinkofliving** and **thinkofliving-frontend**

  **3.2.6. Verify that site is up and running**
  Visit thinkofliving.com and verify that the site is up and running. Take note that CDN cache invalidation can take time.

#### 3.3 Reverting the Backup Region

When the backup region is no longer required, take the following steps to switch traffic back up the primary region and hibernate the backup region.

  - 3.3.1. Update CloudFront origin to switch live traffic to primary region
  - 3.3.2. Invalidate CloudFront cache
  - 3.3.3. Verify that site is up and running
  - 3.3.4. Downscale RDS in backup region
  - 3.3.4. Downscale Elastic Search in backup region
  - 3.3.5. Decrease Fargate task count for Web Frontend, API, and CMS applications in backup region

### 4. Scenario D: Data Loss

#### 4.1 Recovery Procedure
In the event that some critical data is discovered to be missing from the source of truth, use the following procedure for data recovery.

  - 4.1.1. Restore MySQL backup dump file from S3 into Wordpress DB.

## Notes

#### Application

Web:
- File `.env` values `PROD_ASSETS_URL_PREFIX` and `API_URL`
  No changes required for the following as the endpoint is calling CDN. The origin of the CDN will be updated to DR region, so the endpoint remains the same.

- File `src/common/config/img.js`
  - `imgSharpieUrls` no changes required as the endpoint is calling CDN.
  - `imgOriginUrls` no changes required as the endpoint is calling CDN.

- Ensure that all functionality can operate without Elastic Search

API:
- Turn off ES for the build that is deployed to DR region

#### Deployment Pipeline

The BuildKite pipeline `tol-build-deploy-images` should support deployments to DR region in the `Unblock Production` step.

#### Data Backup
Set up daily MySQL dump at 3AM (Kuala Lumpur Time) for the Wordpress database and store the dump file in S3. Configure auto-cleanup of dump files after 30 days old.

#### Infra Diagram
 ![Infra Diagram](assets/TOL-DR-Diagram.png?raw=true "Infra Diagram")

## Appendix

##### TOL Auto Scaling Configurations
| Application | Min Task | Desired Task | Max Task |
| --------| ----------- | ----- |-----|
| tol-frontend-blue-EcsService | 8| 8| 20|
| tol-api-blue-EcsService | 2| 2| 2|
| tol-wordpress-blue-EcsService | 2| 2| 2|

##### TOL CDN Distribution Table

| Distribution ID | Comment | Origin ID | Origin Domain (SG) | Origin Domain (TK) |
| -------- | -------- | -------- | -------- | -------- |
| E1434GUJYD7YYF | thinkofliving-wordpress | Public-Wordpress | wordpress-active.prod.thinkofliving.com | wordpress-active-tk.prod.thinkofliving.com |
|  | | Public-S3 | tol-public-content-prod.s3.amazonaws.com | tol-public-content-prod-tk.s3.amazonaws.com |
| E3E3UNXKAIAKTT | thinkofliving-frontend | Public-Wordpress | wordpress-active.prod.thinkofliving.com | wordpress-active-tk.prod.thinkofliving.com |
|  | | Public-S3 | tol-public-content-prod.s3.amazonaws.com | tol-public-content-prod-tk.s3.amazonaws.com |
|  | | Public-Frontend | frontend-active.prod.thinkofliving.com | frontend-active-tk.prod.thinkofliving.com |
| E2MZT0T360DFAQ | thinkofliving | Public-S3 | tol-public-content-prod.s3.amazonaws.com | tol-public-content-prod-tk.s3.amazonaws.com |

**Distributions**:
 ![CloudFront Distributions](assets/TOL-DR-CF-Distributions.png?raw=true "CloudFront Distributions")

branch_name: refs/heads/master

repo_language: Markdown


text:
<repo_name>Esri/aggregation-viewer-client-feature-layer<file_sep>/README.md
# aggregation-viewer-client-feature-layer

Sample JavaScript Feature and Map Service Layer Aggregation Viewer, using aggregation (lod) queries against Feature or Map Service Layers, aggregating features server side, returning aggregated bins (polygon features) and rendering those aggregation bins client side.

Client side rendering based on server side Flat Hexagon aggregation:

![App](client-flat-hexagon.png?raw=true)

Client side rendering based on server side Square aggregation:

![alt text](client-square.png?raw=true)

Client side rendering based on server side Flat Triangle aggregation:

![alt text](client-flat-triangle.png?raw=true)

Client side Heat Map rendering based on server side Flat Hexagon aggregation:

![alt text](client-flat-hexagon-heat-map.png?raw=true)

## Issues

Find a bug or want to request a new feature?  Please let us know by submitting an issue.

## Contributing

Esri welcomes contributions from anyone and everyone. Please see our [guidelines for contributing](https://github.com/esri/contributing).

## Licensing
Copyright 2017 Esri

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

A copy of the license is available in the repository's [license.txt](license.txt?raw=true) file.


text:
<repo_name>kalp97/User-management-admins-in-PHP-Laravel<file_sep>/tests/Feature/Auth/AdminLoginTest.php
<?php
namespace Tests\Feature\Auth;
use Tests\TestCase;
use Illuminate\Foundation\Testing\WithFaker;
use Illuminate\Foundation\Testing\RefreshDatabase;
class AdminLoginTest extends TestCase
{
    /**
     * A basic test example.
     *
     * @return void
     */
    public function test_admin_can_view_a_login_form()
    {
        $response = $this->get('/admin/login');
        $response->assertSuccessful();
    }
}<file_sep>/README.md
# faq1

To run the FAQ project:

1. git clone https://github.com/kalp97/faq1.git

2. CD into FAQ and run composer install
3. cp .env.example to .env
4. setup database with sqlite or other
(https://laravel.com/docs/5.6/database)

#Epic:

This is a FAQ project where users can ask questions and answers the questions. The new feature I have implemented is the Admin role, where  Admin can LOGIN with the Project Owner provided

* E-mail ID:"<EMAIL>" (without the double quotes)
* Password:"<PASSWORD>"            (without the double quotes)

In this feature, whichever the Users are/were logged into the system, will be tracked by the Admin and will show the history of total Online Users into the system with their login priority and E-mail ID. A simple Welcome page with the type of User(Admin/User) who is either Logged In/Logged Out is showed. The same Login/Logout status is showed in the Admin/User page individually.

#User Stories:

1) Admin can register as a User, but User without knowing the Admin credentials cannot able to login as an Admin.
2) Admin logins and see the history of total users who used the system.
3) Users can edit their profile pages in the profile link.
4) All the users can create, edit and delete questions in the forum.
Heroku Link https://isfinalfaq.herokuapp.com/

languages: Markdown PHP


text:
<repo_name>BurlingtonCodeAcademy/guess-the-number-JenaDurell<file_sep>/compGuessNum.js
//computer guesses number
const {ask} = require('./ask');

async function startCompGuessNum() {
  //range
  let high = (process.argv[2] || 100);
  let low = 1;

  console.log(`Let's play a game where you (human) think up a number and I (computer) try to guess it. \nPlease think of a number between 1 and ${high} (inclusive).`);
  let questionText = "What is your secret number?\nI won't peek, I promise...\n";
  let secretNumber = await ask(questionText);
  console.log('You entered: ' + secretNumber);

  //start of game guess count
  let guessCount = 0;
  //computer guess
  let computerGuess = Math.floor((high - low) / 2) + low;
  //myAnswer is Y or N
  let myAnswer = await ask(`Is it ${computerGuess}, yes (y) or no (n)?`);
  guessCount++

  //Cheat detector guessing above
  function highCheatTest(compGuess, limit) {
    if (compGuess >= limit) {
      console.log(`But you said it was lower than ${parseInt(limit + 1)}! I don't play with cheaters, goodbye!`);
      process.exit();
    } else {
      return true
    }
  };
  // Cheat detector guessing below
  function lowCheatTest(compGuess, limit) {
    if (compGuess <= limit) {
      console.log(`But you said it was higher than ${parseInt(limit - 1)}! I don't play with cheaters, goodbye!`);
      process.exit();
    } else {
      return true
    }
  };
  //wrong guess answered
  while (myAnswer.toLowerCase() === 'n') {
    let highLow = await ask('Is it higher (h) or lower (l)?')
    if (highLow.toLowerCase() === 'h' && highCheatTest(computerGuess, high)) {
      low = computerGuess + 1;
    } else if (highLow.toLowerCase() === 'l' && lowCheatTest(computerGuess, low)) {
      high = computerGuess - 1;
    } else if (highLow.toLowerCase() !== 'h' && highLow.toLowerCase() !== 'l') {
      console.log("I'm sorry, what?!?!?")
    }
    computerGuess = Math.floor((high - low) / 2) + low;
    myAnswer = await ask(`Is it ${computerGuess}, yes (y) or no (n)?`);
    guessCount++
  }
  //correct guess answered
  if (myAnswer.toLowerCase() === 'y') {
    //console.log using ternary operator for correct tenses
    console.log(`Yes! I got you number and it only took ${guessCount} guess${guessCount > 1 ? "es" : ""} buddy! \n \n`);

  }

}
module.exports = {startCompGuessNum}<file_sep>/ask.js
const readline = require('readline');
const rl = readline.createInterface(process.stdin, process.stdout);

function ask(questionText) {
    return new Promise((resolve, reject) => {
        rl.question(questionText, resolve);
    });
}

//share file within project
module.exports = { ask }<file_sep>/humGuessNum.js
//Human guess start game
const {ask} = require('./ask');

async function startHumGuessNum() {
    //range
    let high = (process.argv[2] || 100);
    let low = 1;

    console.log(`Let's play a game where I (computer) think up a number and you (human) try to guess it. \nI am thinking of a number between 1 and ${high} (inclusive).`);
    let secretNumber = randomNum(low, high);
    let questionText = "What do you think it could be?\n";
    let myGuess = await ask(questionText);
    myGuess = parseInt(myGuess);

    //guess count
    let guessCount = 0;
    guessCount++

    //correct guess answered
    if (myGuess === secretNumber) {
      myGuess = parseInt(myGuess);
      win();
    }

    while (myGuess != secretNumber) {
      //wrong guess answered and it's too high
      if (myGuess > secretNumber) {
        console.log(`Nope! ${myGuess} is too high! `);
        myGuess = await ask("what do you think it could be? ");
        myGuess = parseInt(myGuess);
        //wrong guess answered and it's too low
      } else {
        console.log(`Nope! ${myGuess} is too low! `);
        myGuess = await ask("What do you think it could be? ");
        myGuess = parseInt(myGuess);
      }
      guessCount++
    }
    win();

    function win() {
      //console.log using ternary operator for correct tenses
      console.log(`Yes! You got my number and it only took ${guessCount} guess${guessCount > 1 ? "es" : ""}! Strong work, Wise One!\n \n`);

    }

    //random number generator
    function randomNum(min, max) {
      return Math.floor(Math.random() * (max - min + 1) + min);
    }

  }

module.exports = {startHumGuessNum}

repo_language: JavaScript


text:
<repo_name>huanglitao0912/Swords<file_sep>/README.md
# Swords
The Product By The Swords Team


text:
<file_sep># OpenWeatherMap API Key
api_key = "575f4e0d1431e778a2f1ac03d4f8e06e"

repo_language: Python


text:
<file_sep># Site settings
title: hwi
description: 개발 공부중
baseurl: "" # the subpath of your site, e.g. /blog
url: "https://ssohwi.github.io/" # the base hostname & protocol for your site e.g. http://willianjusten.com.br

# User settings
username: Lee SoHwi
user_description: Student
user_title: Software Engineer
email: <EMAIL>
# twitter_username: lorem_ipsum
# github_username:  lorem_ipsum
# gplus_username:  lorem_ipsum

exclude: ['package.json', 'src' , 'node_modules']

repo_language: YAML