-{-# LANGUAGE RankNTypes, TypeFamilies, BangPatterns, CPP #-}
-#if __GLASGOW_HASKELL__ >= 702
-{-# LANGUAGE Trustworthy #-}
-#endif
-{-| This module provides @pipes@ utilities for \"text streams\", which are
- streams of 'Text' chunks. The individual chunks are uniformly @strict@, but
- a 'Producer' can be converted to and from lazy 'Text's, though this is generally
- unwise. Where pipes IO replaces lazy IO, 'Producer Text m r' replaces lazy 'Text'.
- An 'IO.Handle' can be associated with a 'Producer' or 'Consumer' according as it is read or written to.
-
- To stream to or from 'IO.Handle's, one can use 'fromHandle' or 'toHandle'. For
- example, the following program copies a document from one file to another:
-
-> import Pipes
-> import qualified Data.Text.Pipes as Text
-> import System.IO
->
-> main =
-> withFile "inFile.txt" ReadMode $ \hIn ->
-> withFile "outFile.txt" WriteMode $ \hOut ->
-> runEffect $ Text.fromHandle hIn >-> Text.toHandle hOut
-
-To stream from files, the following is perhaps more Prelude-like (note that it uses Pipes.Safe):
-
-> import Pipes
-> import qualified Data.Text.Pipes as Text
-> import Pipes.Safe
->
-> main = runSafeT $ runEffect $ Text.readFile "inFile.txt" >-> Text.writeFile "outFile.txt"
-
- You can stream to and from 'stdin' and 'stdout' using the predefined 'stdin'
- and 'stdout' proxies, as with the following \"echo\" program:
-
-> main = runEffect $ Text.stdin >-> Text.stdout
-
- You can also translate pure lazy 'TL.Text's to and from proxies:
-
-> main = runEffect $ Text.fromLazy (TL.pack "Hello, world!\n") >-> Text.stdout
+{-# LANGUAGE RankNTypes, TypeFamilies, BangPatterns, Trustworthy #-}
+
+{-| This package provides @pipes@ utilities for \'text streams\', which are
+ streams of 'Text' chunks. The individual chunks are uniformly @strict@, and you
+ will generally want @Data.Text@ in scope. But the type @Producer Text m r@ is
+ in some ways the pipes equivalent of the lazy @Text@ type.
+
+ This module provides many functions equivalent in one way or another to
+ the 'pure' functions in
+ <https://hackage.haskell.org/package/text-1.1.0.0/docs/Data-Text-Lazy.html Data.Text.Lazy>.
+ They transform, divide, group and fold text streams. Though @Producer Text m r@
+ is \'effectful\' Text, functions
+ in this module are \'pure\' in the sense that they are uniformly monad-independent.
+ Simple IO operations are defined in @Pipes.Text.IO@ -- as lazy IO @Text@
+ operations are in @Data.Text.Lazy.IO@. Interoperation with @ByteString@
+ is provided in @Pipes.Text.Encoding@, which parallels @Data.Text.Lazy.Encoding@.
+
+ The Text type exported by @Data.Text.Lazy@ is basically '[Text]'. The implementation
+ is arranged so that the individual strict 'Text' chunks are kept to a reasonable size;
+ the user is not aware of the divisions between the connected 'Text' chunks.
+ So also here: the functions in this module are designed to operate on streams that
+ are insensitive to text boundaries. This means that they may freely split
+ text into smaller texts and /discard empty texts/. However, the objective is
+ that they should /never concatenate texts/ in order to provide strict upper
+ bounds on memory usage.
- In addition, this module provides many functions equivalent to lazy
- 'Text' functions so that you can transform or fold text streams. For
- example, to stream only the first three lines of 'stdin' to 'stdout' you
+ For example, to stream only the first three lines of 'stdin' to 'stdout' you
might write:
> import Pipes
> import qualified Pipes.Text as Text
-> import qualified Pipes.Parse as Parse
->
+> import qualified Pipes.Text.IO as Text
+> import Pipes.Group
+> import Lens.Family
+>
> main = runEffect $ takeLines 3 Text.stdin >-> Text.stdout
-> where
-> takeLines n = Text.unlines . Parse.takeFree n . Text.lines
+> where
+> takeLines n = Text.unlines . takes' n . view Text.lines
+> -- or equivalently:
+> -- takeLines n = over Text.lines (takes' n)
The above program will never bring more than one chunk of text (~ 32 KB) into
memory, no matter how long the lines are.
+
+ As this example shows, one superficial difference from @Data.Text.Lazy@
+ is that many of the operations, like 'lines',
+ are \'lensified\'; this has a number of advantages where it is possible, in particular
+ it facilitates their use with 'Parser's of Text (in the general
+ <http://hackage.haskell.org/package/pipes-parse-3.0.1/docs/Pipes-Parse-Tutorial.html pipes-parse>
+ sense.)
+ Each such expression, e.g. 'lines', 'chunksOf' or 'splitAt', reduces to the
+ intuitively corresponding function when used with @view@ or @(^.)@.
+
+ A more important difference the example reveals is in the types closely associated with
+ the central type, @Producer Text m r@. In @Data.Text@ and @Data.Text.Lazy@
+ we find functions like
+
+> splitAt :: Int -> Text -> (Text, Text)
+> lines :: Int -> Text -> [Text]
+> chunksOf :: Int -> Text -> [Text]
+
+ which relate a Text with a pair or list of Texts. The corresponding functions here (taking
+ account of \'lensification\') are
+
+> view . splitAt :: (Monad m, Integral n)
+> => n -> Producer Text m r -> Producer Text.Text m (Producer Text.Text m r)
+> view lines :: Monad m => Producer Text m r -> FreeT (Producer Text m) m r
+> view . chunksOf :: (Monad m, Integral n) => n -> Producer Text m r -> FreeT (Producer Text m) m r
+
+ In the type @Producer Text m (Producer Text m r)@ the second
+ element of the \'pair\' of of \'effectful Texts\' cannot simply be retrieved
+ with 'snd'. This is an \'effectful\' pair, and one must work through the effects
+ of the first element to arrive at the second. Similarly in @FreeT (Producer Text m) m r@,
+ which corresponds with @[Text]@, on cannot simply drop 10 Producers and take the others;
+ we can only get to the ones we want to take by working through their predecessors.
+
+ Some of the types may be more readable if you imagine that we have introduced
+ our own type synonyms
+
+> type Text m r = Producer T.Text m r
+> type Texts m r = FreeT (Producer T.Text m) m r
+
+ Then we would think of the types above as
+
+> view . splitAt :: (Monad m, Integral n) => n -> Text m r -> Text m (Text m r)
+> view lines :: (Monad m) => Text m r -> Texts m r
+> view . chunksOf :: (Monad m, Integral n) => n -> Text m r -> Texts m r
+
+ which brings one closer to the types of the similar functions in @Data.Text.Lazy@
- Note that functions in this library are designed to operate on streams that
- are insensitive to text boundaries. This means that they may freely split
- text into smaller texts, /discard empty texts/. However, apart from the
- special case of 'concatMap', they will /never concatenate texts/ in order
- to provide strict upper bounds on memory usage -- with the single exception of 'concatMap'.
-}
module Pipes.Text (
-- * Producers
- fromLazy
- , stdin
- , fromHandle
- , readFile
-
- -- * Consumers
- , stdout
- , toHandle
- , writeFile
+ fromLazy
-- * Pipes
, map
, dropWhile
, filter
, scan
- , encodeUtf8
, pack
, unpack
, toCaseFold
, count
-- * Primitive Character Parsers
- -- $parse
, nextChar
, drawChar
, unDrawChar
, group
, word
, line
-
- -- * Decoding Lenses
- , decodeUtf8
- , codec
-
- -- * Codecs
- , utf8
- , utf16_le
- , utf16_be
- , utf32_le
- , utf32_be
-
- -- * Other Decoding/Encoding Functions
- , decodeIso8859_1
- , decodeAscii
- , encodeIso8859_1
- , encodeAscii
-- * FreeT Splitters
, chunksOf
, splitsWith
, splits
--- , groupsBy
--- , groups
+ , groupsBy
+ , groups
, lines
, words
, unlines
, unwords
- -- * Re-exports
+ -- * Re-exports
-- $reexports
- , Decoding(..)
- , streamDecodeUtf8
- , decodeSomeUtf8
- , Codec(..)
- , TextException(..)
, module Data.ByteString
, module Data.Text
, module Data.Profunctor
- , module Data.Word
, module Pipes.Parse
, module Pipes.Group
) where
-import Control.Exception (throwIO, try)
import Control.Applicative ((<*))
-import Control.Monad (liftM, unless, join)
+import Control.Monad (liftM, join)
import Control.Monad.Trans.State.Strict (StateT(..), modify)
-import Data.Monoid ((<>))
import qualified Data.Text as T
-import qualified Data.Text.IO as T
-import qualified Data.Text.Encoding as TE
-import qualified Data.Text.Encoding.Error as TE
import Data.Text (Text)
import qualified Data.Text.Lazy as TL
-import qualified Data.Text.Lazy.IO as TL
import Data.Text.Lazy.Internal (foldrChunks, defaultChunkSize)
-import Data.ByteString.Unsafe (unsafeTake, unsafeDrop)
import Data.ByteString (ByteString)
-import qualified Data.ByteString as B
-import qualified Data.ByteString.Char8 as B8
-import Data.Char (ord, isSpace)
import Data.Functor.Constant (Constant(Constant, getConstant))
import Data.Functor.Identity (Identity)
import Data.Profunctor (Profunctor)
import qualified Data.Profunctor
-import qualified Data.List as List
-import Foreign.C.Error (Errno(Errno), ePIPE)
-import qualified GHC.IO.Exception as G
import Pipes
-import qualified Pipes.ByteString as PB
-import qualified Pipes.Text.Internal as PI
-import Pipes.Text.Internal
-import Pipes.Core (respond, Server')
import Pipes.Group (concats, intercalates, FreeT(..), FreeF(..))
import qualified Pipes.Group as PG
import qualified Pipes.Parse as PP
import Pipes.Parse (Parser)
-import qualified Pipes.Safe.Prelude as Safe
-import qualified Pipes.Safe as Safe
-import Pipes.Safe (MonadSafe(..), Base(..))
import qualified Pipes.Prelude as P
-import qualified System.IO as IO
import Data.Char (isSpace)
import Data.Word (Word8)
fromLazy = foldrChunks (\e a -> yield e >> a) (return ())
{-# INLINE fromLazy #-}
--- | Stream text from 'stdin'
-stdin :: MonadIO m => Producer Text m ()
-stdin = fromHandle IO.stdin
-{-# INLINE stdin #-}
-
-{-| Convert a 'IO.Handle' into a text stream using a text size
- determined by the good sense of the text library; note that this
- is distinctly slower than @decideUtf8 (Pipes.ByteString.fromHandle h)@
- but uses the system encoding and has other `Data.Text.IO` features
--}
-
-fromHandle :: MonadIO m => IO.Handle -> Producer Text m ()
-fromHandle h = go where
- go = do txt <- liftIO (T.hGetChunk h)
- unless (T.null txt) ( do yield txt
- go )
-{-# INLINABLE fromHandle#-}
-
-
-{-| Stream text from a file in the simple fashion of @Data.Text.IO@
-
->>> runSafeT $ runEffect $ Text.readFile "hello.hs" >-> Text.map toUpper >-> hoist lift Text.stdout
-MAIN = PUTSTRLN "HELLO WORLD"
--}
-
-readFile :: MonadSafe m => FilePath -> Producer Text m ()
-readFile file = Safe.withFile file IO.ReadMode fromHandle
-{-# INLINE readFile #-}
-
-
-{-| Stream text to 'stdout'
-
- Unlike 'toHandle', 'stdout' gracefully terminates on a broken output pipe.
-
- Note: For best performance, it might be best just to use @(for source (liftIO . putStr))@
- instead of @(source >-> stdout)@ .
--}
-stdout :: MonadIO m => Consumer' Text m ()
-stdout = go
- where
- go = do
- txt <- await
- x <- liftIO $ try (T.putStr txt)
- case x of
- Left (G.IOError { G.ioe_type = G.ResourceVanished
- , G.ioe_errno = Just ioe })
- | Errno ioe == ePIPE
- -> return ()
- Left e -> liftIO (throwIO e)
- Right () -> go
-{-# INLINABLE stdout #-}
-
-
-{-| Convert a text stream into a 'Handle'
-
- Note: again, for best performance, where possible use
- @(for source (liftIO . hPutStr handle))@ instead of @(source >-> toHandle handle)@.
--}
-toHandle :: MonadIO m => IO.Handle -> Consumer' Text m r
-toHandle h = for cat (liftIO . T.hPutStr h)
-{-# INLINABLE toHandle #-}
-
-{-# RULES "p >-> toHandle h" forall p h .
- p >-> toHandle h = for p (\txt -> liftIO (T.hPutStr h txt))
- #-}
-
-
--- | Stream text into a file. Uses @pipes-safe@.
-writeFile :: (MonadSafe m) => FilePath -> Consumer' Text m ()
-writeFile file = Safe.withFile file IO.WriteMode toHandle
-{-# INLINE writeFile #-}
-
type Lens' a b = forall f . Functor f => (b -> f b) -> (a -> f a)
p >-> concatMap f = for p (\txt -> yield (T.concatMap f txt))
#-}
--- | Transform a Pipe of 'Text' into a Pipe of 'ByteString's using UTF-8
--- encoding; @encodeUtf8 = Pipes.Prelude.map TE.encodeUtf8@ so more complex
--- encoding pipes can easily be constructed with the functions in @Data.Text.Encoding@
-encodeUtf8 :: Monad m => Pipe Text ByteString m r
-encodeUtf8 = P.map TE.encodeUtf8
-{-# INLINEABLE encodeUtf8 #-}
-
-{-# RULES "p >-> encodeUtf8" forall p .
- p >-> encodeUtf8 = for p (\txt -> yield (TE.encodeUtf8 txt))
- #-}
-- | Transform a Pipe of 'String's into one of 'Text' chunks
pack :: Monad m => Pipe String Text m r
scan
:: (Monad m)
=> (Char -> Char -> Char) -> Char -> Pipe Text Text m r
-scan step begin = go begin
+scan step begin = do
+ yield (T.singleton begin)
+ go begin
where
go c = do
txt <- await
let txt' = T.scanl step c txt
c' = T.last txt'
- yield txt'
+ yield (T.tail txt')
go c'
{-# INLINABLE scan #-}
Just c -> Just (min c (T.minimum txt))
{-# INLINABLE minimum #-}
-
-- | Find the first element in the stream that matches the predicate
find
:: (Monad m)
{-# INLINABLE count #-}
-{-| Consume the first character from a stream of 'Text'
+-- | Consume the first character from a stream of 'Text'
+--
+-- 'next' either fails with a 'Left' if the 'Producer' has no more characters or
+-- succeeds with a 'Right' providing the next character and the remainder of the
+-- 'Producer'.
- 'next' either fails with a 'Left' if the 'Producer' has no more characters or
- succeeds with a 'Right' providing the next character and the remainder of the
- 'Producer'.
--}
nextChar
:: (Monad m)
=> Producer Text m r
Just (c, txt') -> return (Right (c, yield txt' >> p'))
{-# INLINABLE nextChar #-}
-{-| Draw one 'Char' from a stream of 'Text', returning 'Left' if the
- 'Producer' is empty
--}
+-- | Draw one 'Char' from a stream of 'Text', returning 'Left' if the 'Producer' is empty
+
drawChar :: (Monad m) => Parser Text m (Maybe Char)
drawChar = do
x <- PP.draw
> Left _ -> return ()
> Right c -> unDrawChar c
> return x
+
-}
+
peekChar :: (Monad m) => Parser Text m (Maybe Char)
peekChar = do
x <- drawChar
{-# INLINABLE isEndOfChars #-}
-{- | An improper lens into a stream of 'ByteString' expected to be UTF-8 encoded; the associated
- stream of Text ends by returning a stream of ByteStrings beginning at the point of failure.
- -}
-
-decodeUtf8 :: Monad m => Lens' (Producer ByteString m r)
- (Producer Text m (Producer ByteString m r))
-decodeUtf8 k p0 = fmap (\p -> join (for p (yield . TE.encodeUtf8)))
- (k (go B.empty PI.streamDecodeUtf8 p0)) where
- go !carry dec0 p = do
- x <- lift (next p)
- case x of Left r -> return (if B.null carry
- then return r -- all bytestring input was consumed
- else (do yield carry -- a potentially valid fragment remains
- return r))
-
- Right (chunk, p') -> case dec0 chunk of
- PI.Some text carry2 dec -> do yield text
- go carry2 dec p'
- PI.Other text bs -> do yield text
- return (do yield bs -- an invalid blob remains
- p')
-{-# INLINABLE decodeUtf8 #-}
-
-
-- | Splits a 'Producer' after the given number of characters
splitAt
:: (Monad m, Integral n)
{-# INLINABLE splitAt #-}
-{-| Split a text stream in two, where the first text stream is the longest
- consecutive group of text that satisfy the predicate
--}
+-- | Split a text stream in two, producing the longest
+-- consecutive group of characters that satisfies the predicate
+-- and returning the rest
+
span
:: (Monad m)
=> (Char -> Bool)
return (yield suffix >> p')
{-# INLINABLE span #-}
-{-| Split a text stream in two, where the first text stream is the longest
+{-| Split a text stream in two, producing the longest
consecutive group of characters that don't satisfy the predicate
-}
break
unwords = intercalate (yield $ T.singleton ' ')
{-# INLINABLE unwords #-}
-{- $parse
- The following parsing utilities are single-character analogs of the ones found
- @pipes-parse@.
--}
{- $reexports
@Pipes.Parse@ re-exports 'input', 'concat', 'FreeT' (the type) and the 'Parse' synonym.
-}
-{- | Use a 'Codec' as a pipes-style 'Lens' into a byte stream; the available 'Codec' s are
- 'utf8', 'utf16_le', 'utf16_be', 'utf32_le', 'utf32_be' . The 'Codec' concept and the
- individual 'Codec' definitions follow the enumerator and conduit libraries.
-
- Utf8 is handled differently in this library -- without the use of 'unsafePerformIO' &co
- to catch 'Text' exceptions; but the same 'mypipe ^. codec utf8' interface can be used.
- 'mypipe ^. decodeUtf8' should be the same, but has a somewhat more direct and thus perhaps
- better implementation.
-
- -}
-codec :: Monad m => Codec -> Lens' (Producer ByteString m r) (Producer Text m (Producer ByteString m r))
-codec (Codec _ enc dec) k p0 = fmap (\p -> join (for p (yield . fst . enc)))
- (k (decoder (dec B.empty) p0) ) where
- decoder :: Monad m => PI.Decoding -> Producer ByteString m r -> Producer Text m (Producer ByteString m r)
- decoder !d p0 = case d of
- PI.Other txt bad -> do yield txt
- return (do yield bad
- p0)
- PI.Some txt extra dec -> do yield txt
- x <- lift (next p0)
- case x of Left r -> return (do yield extra
- return r)
- Right (chunk,p1) -> decoder (dec chunk) p1
-
-{- | ascii and latin encodings only represent a small fragment of 'Text'; thus we cannot
- use the pipes 'Lens' style to work with them. Rather we simply define functions
- each way.
-
- 'encodeAscii' : Reduce as much of your stream of 'Text' actually is ascii to a byte stream,
- returning the rest of the 'Text' at the first non-ascii 'Char'
--}
-encodeAscii :: Monad m => Producer Text m r -> Producer ByteString m (Producer Text m r)
-encodeAscii = go where
- go p = do echunk <- lift (next p)
- case echunk of
- Left r -> return (return r)
- Right (chunk, p') ->
- if T.null chunk
- then go p'
- else let (safe, unsafe) = T.span (\c -> ord c <= 0x7F) chunk
- in do yield (B8.pack (T.unpack safe))
- if T.null unsafe
- then go p'
- else return $ do yield unsafe
- p'
-{- | Reduce as much of your stream of 'Text' actually is iso8859 or latin1 to a byte stream,
- returning the rest of the 'Text' upon hitting any non-latin 'Char'
- -}
-encodeIso8859_1 :: Monad m => Producer Text m r -> Producer ByteString m (Producer Text m r)
-encodeIso8859_1 = go where
- go p = do etxt <- lift (next p)
- case etxt of
- Left r -> return (return r)
- Right (txt, p') ->
- if T.null txt
- then go p'
- else let (safe, unsafe) = T.span (\c -> ord c <= 0xFF) txt
- in do yield (B8.pack (T.unpack safe))
- if T.null unsafe
- then go p'
- else return $ do yield unsafe
- p'
-
-{- | Reduce a byte stream to a corresponding stream of ascii chars, returning the
- unused 'ByteString' upon hitting an un-ascii byte.
- -}
-decodeAscii :: Monad m => Producer ByteString m r -> Producer Text m (Producer ByteString m r)
-decodeAscii = go where
- go p = do echunk <- lift (next p)
- case echunk of
- Left r -> return (return r)
- Right (chunk, p') ->
- if B.null chunk
- then go p'
- else let (safe, unsafe) = B.span (<= 0x7F) chunk
- in do yield (T.pack (B8.unpack safe))
- if B.null unsafe
- then go p'
- else return $ do yield unsafe
- p'
-
-{- | Reduce a byte stream to a corresponding stream of ascii chars, returning the
- unused 'ByteString' upon hitting the rare un-latinizable byte.
- -}
-decodeIso8859_1 :: Monad m => Producer ByteString m r -> Producer Text m (Producer ByteString m r)
-decodeIso8859_1 = go where
- go p = do echunk <- lift (next p)
- case echunk of
- Left r -> return (return r)
- Right (chunk, p') ->
- if B.null chunk
- then go p'
- else let (safe, unsafe) = B.span (<= 0xFF) chunk
- in do yield (T.pack (B8.unpack safe))
- if B.null unsafe
- then go p'
- else return $ do yield unsafe
- p'
-
-
-
-
-
\ No newline at end of file
+
projects / github / fretlink / text-pipes.git / blobdiff