diff options
Diffstat (limited to 'Pipes/Text/Internal')
-rw-r--r-- | Pipes/Text/Internal/Codec.hs | 16 | ||||
-rw-r--r-- | Pipes/Text/Internal/Decoding.hs | 23 |
2 files changed, 20 insertions, 19 deletions
diff --git a/Pipes/Text/Internal/Codec.hs b/Pipes/Text/Internal/Codec.hs index 63cbd74..075a152 100644 --- a/Pipes/Text/Internal/Codec.hs +++ b/Pipes/Text/Internal/Codec.hs | |||
@@ -4,17 +4,15 @@ | |||
4 | -- Copyright: 2014 Michael Thompson, 2011 Michael Snoyman, 2010-2011 John Millikin | 4 | -- Copyright: 2014 Michael Thompson, 2011 Michael Snoyman, 2010-2011 John Millikin |
5 | -- License: MIT | 5 | -- License: MIT |
6 | -- This Parts of this code were taken from enumerator and conduits, and adapted for pipes | 6 | -- This Parts of this code were taken from enumerator and conduits, and adapted for pipes |
7 | {- | This module follows the model of the enumerator and conduits libraries, and defines | 7 | |
8 | 'Codec' s for various encodings. Note that we do not export a 'Codec' for ascii and | 8 | -- This module follows the model of the enumerator and conduits libraries, and defines |
9 | iso8859_1. A 'Lens' in the sense of the pipes library cannot be defined for these, so | 9 | -- 'Codec' s for various encodings. Note that we do not export a 'Codec' for ascii and |
10 | special functions appear in @Pipes.Text@ | 10 | -- iso8859_1. A 'Lens' in the sense of the pipes library cannot be defined for these, so |
11 | -} | 11 | -- special functions appear in @Pipes.Text@ |
12 | |||
12 | 13 | ||
13 | module Pipes.Text.Internal.Codec | 14 | module Pipes.Text.Internal.Codec |
14 | ( Decoding(..) | 15 | ( Codec(..) |
15 | , streamDecodeUtf8 | ||
16 | , decodeSomeUtf8 | ||
17 | , Codec(..) | ||
18 | , TextException(..) | 16 | , TextException(..) |
19 | , utf8 | 17 | , utf8 |
20 | , utf16_le | 18 | , utf16_le |
diff --git a/Pipes/Text/Internal/Decoding.hs b/Pipes/Text/Internal/Decoding.hs index 4b4bbe6..b5d928a 100644 --- a/Pipes/Text/Internal/Decoding.hs +++ b/Pipes/Text/Internal/Decoding.hs | |||
@@ -2,11 +2,10 @@ | |||
2 | {-# LANGUAGE GeneralizedNewtypeDeriving, MagicHash, UnliftedFFITypes #-} | 2 | {-# LANGUAGE GeneralizedNewtypeDeriving, MagicHash, UnliftedFFITypes #-} |
3 | {-# LANGUAGE DeriveDataTypeable, RankNTypes #-} | 3 | {-# LANGUAGE DeriveDataTypeable, RankNTypes #-} |
4 | 4 | ||
5 | {- | | 5 | -- This module lifts assorted materials from Brian O'Sullivan's text package |
6 | This module lifts assorted materials from Brian O'Sullivan's text package | 6 | -- especially @Data.Text.Encoding@ in order to define a pipes-appropriate |
7 | especially @Data.Text.Encoding@ in order to define a pipes-appropriate | 7 | -- 'streamDecodeUtf8' |
8 | 'streamDecodeUtf8' | 8 | |
9 | -} | ||
10 | module Pipes.Text.Internal.Decoding | 9 | module Pipes.Text.Internal.Decoding |
11 | ( Decoding(..) | 10 | ( Decoding(..) |
12 | , streamDecodeUtf8 | 11 | , streamDecodeUtf8 |
@@ -43,9 +42,13 @@ import Data.Maybe (catMaybes) | |||
43 | 42 | ||
44 | 43 | ||
45 | 44 | ||
46 | -- | A stream oriented decoding result. Distinct from the similar type in @Data.Text.Encoding@ | 45 | -- A stream oriented decoding result. Distinct from the similar type in Data.Text.Encoding |
47 | data Decoding = Some Text ByteString (ByteString -> Decoding) -- | Text, continuation and any undecoded fragment. | 46 | |
48 | | Other Text ByteString -- | Text followed by an undecodable ByteString | 47 | data Decoding = Some Text ByteString (ByteString -> Decoding) |
48 | -- Text, continuation and any undecoded fragment. | ||
49 | | Other Text ByteString | ||
50 | -- Text followed by an undecodable ByteString | ||
51 | |||
49 | instance Show Decoding where | 52 | instance Show Decoding where |
50 | showsPrec d (Some t bs _) = showParen (d > prec) $ | 53 | showsPrec d (Some t bs _) = showParen (d > prec) $ |
51 | showString "Some " . showsPrec prec' t . | 54 | showString "Some " . showsPrec prec' t . |
@@ -61,7 +64,7 @@ instance Show Decoding where | |||
61 | newtype CodePoint = CodePoint Word32 deriving (Eq, Show, Num, Storable) | 64 | newtype CodePoint = CodePoint Word32 deriving (Eq, Show, Num, Storable) |
62 | newtype DecoderState = DecoderState Word32 deriving (Eq, Show, Num, Storable) | 65 | newtype DecoderState = DecoderState Word32 deriving (Eq, Show, Num, Storable) |
63 | 66 | ||
64 | -- | Resolve a 'ByteString' into 'Text' and a continuation that can handle further 'ByteStrings'. | 67 | -- Resolve a 'ByteString' into 'Text' and a continuation that can handle further 'ByteStrings'. |
65 | streamDecodeUtf8 :: ByteString -> Decoding | 68 | streamDecodeUtf8 :: ByteString -> Decoding |
66 | streamDecodeUtf8 = decodeChunkUtf8 B.empty 0 0 | 69 | streamDecodeUtf8 = decodeChunkUtf8 B.empty 0 0 |
67 | where | 70 | where |
@@ -95,7 +98,7 @@ streamDecodeUtf8 = decodeChunkUtf8 B.empty 0 0 | |||
95 | {-# INLINE decodeChunkUtf8 #-} | 98 | {-# INLINE decodeChunkUtf8 #-} |
96 | {-# INLINE streamDecodeUtf8 #-} | 99 | {-# INLINE streamDecodeUtf8 #-} |
97 | 100 | ||
98 | -- | Resolve a ByteString into an initial segment of intelligible 'Text' and whatever is unintelligble | 101 | -- Resolve a ByteString into an initial segment of intelligible 'Text' and whatever is unintelligble |
99 | decodeSomeUtf8 :: ByteString -> (Text, ByteString) | 102 | decodeSomeUtf8 :: ByteString -> (Text, ByteString) |
100 | decodeSomeUtf8 bs@(PS fp off len) = runST $ do | 103 | decodeSomeUtf8 bs@(PS fp off len) = runST $ do |
101 | dest <- A.new (len+1) | 104 | dest <- A.new (len+1) |