scaffolder/src/Text/Edifact/Fetcher/Composites.hs

   1 {-# LANGUAGE OverloadedStrings #-}
   2
   3 module Text.Edifact.Fetcher.Composites
   4   ( fetchComposites
   5   , compositesDirectory
   6   ) where
   7
   8 import           Text.Edifact.Fetcher.Commons
   9 import           Text.Edifact.Scaffolder.Commons
  10
  11 import           Text.Edifact.Scaffolder.Composites.Specification (listSimples)
  12
  13 import           Data.List                                        as L (nub,
  14                                                                         sort)
  15 import           Data.Text.Encoding                               as TE (decodeUtf8)
  16 import           Formatting
  17
  18 compositesDirectory :: FilePath
  19 compositesDirectory = "composites"
  20
  21 fetchComposites :: ([CompositeCode], [SimpleCode]) -> Fetcher [SimpleCode]
  22 fetchComposites (composites, segments) =
  23   let compactSimpleCodes = L.nub . L.sort . mappend segments . concat
  24   in compactSimpleCodes <$> traverse (retry 3 . fetchComposite) composites
  25
  26 fetchComposite :: CompositeCode -> Fetcher [SimpleCode]
  27 fetchComposite code = do
  28   say ("Fetching composite " % fCompositeCode) code
  29   url <- getUrl ("/trcd/trcd" % fCompositeCodeLower % ".htm") code
  30   outputFile <- getOutputFile (fCompositeCodeLower % ".txt") compositesDirectory code
  31   htmlToFileWithParser url TE.decodeUtf8 outputFile (snd <$> listSimples)