scaffolder/src/Text/Edifact/Fetcher/Segments.hs

   1 {-# LANGUAGE OverloadedStrings #-}
   2
   3 module Text.Edifact.Fetcher.Segments
   4   ( fetchSegments
   5   , segmentsDirectory
   6   ) where
   7
   8 import           Text.Edifact.Fetcher.Commons
   9 import           Text.Edifact.Scaffolder.Commons
  10
  11 import           Text.Edifact.Scaffolder.Segments.Specification (listCompositesAndSimples)
  12 import           Text.Edifact.Scaffolder.Segments.Types         (Element,
  13                                                                  getComposite,
  14                                                                  getSimple)
  15
  16 import           Data.Bifunctor                                 (bimap)
  17 import           Data.List                                      as L (nub,
  18                                                                       partition,
  19                                                                       sort)
  20 import           Data.Maybe                                     (isJust,
  21                                                                  mapMaybe)
  22 import           Data.Text.Encoding                             as TE (decodeUtf8)
  23 import           Formatting
  24
  25 segmentsDirectory :: FilePath
  26 segmentsDirectory = "segments"
  27
  28 fetchSegments :: [SegmentCode] -> Fetcher ([CompositeCode], [SimpleCode])
  29 fetchSegments = fmap (partitionElements . L.nub . L.sort . concat) . traverse (retry 3 . fetchSegment)
  30
  31 partitionElements :: [Element] -> ([CompositeCode], [SimpleCode])
  32 partitionElements =
  33   let isComposite = isJust . getComposite
  34   in bimap (mapMaybe getComposite) (mapMaybe getSimple) . partition isComposite
  35
  36 fetchSegment :: SegmentCode -> Fetcher [Element]
  37 fetchSegment code = do
  38   say ("Fetching segment " % fSegmentCode) code
  39   url <- getUrl ("/trsd/trsd" % fSegmentCodeLower % ".htm") code
  40   outputFile <- getOutputFile (fSegmentCodeLower % ".txt") segmentsDirectory code
  41   htmlToFileWithParser url TE.decodeUtf8 outputFile (snd <$> listCompositesAndSimples)