tagstream-conduit-0.5.5.3/0000755000000000000000000000000012402232134013473 5ustar0000000000000000tagstream-conduit-0.5.5.3/tagstream-conduit.cabal0000644000000000000000000000430712402232134020115 0ustar0000000000000000Name: tagstream-conduit Version: 0.5.5.3 Synopsis: streamlined html tag parser Description: Tag-stream is a library for parsing HTML//XML to a token stream. It can parse unstructured and malformed HTML from the web. It also provides an Enumeratee which can parse streamline html, which means it consumes constant memory. You can start from the `tests/Tests.hs` module to see what it can do. Homepage: http://github.com/yihuang/tagstream-conduit License: BSD3 License-file: LICENSE Author: yihuang Maintainer: yi.codeplayer@gmail.com Category: Web, Conduit Build-type: Simple Extra-source-files: tests/Tests.hs , Highlight.hs , Parse.hs , HighlightText.hs , ParseText.hs Cabal-version: >=1.8 source-repository head type: git location: git://github.com/yihuang/tagstream-conduit Library GHC-Options: -Wall -O2 Exposed-modules: Text.HTML.TagStream , Text.HTML.TagStream.ByteString , Text.HTML.TagStream.Text , Text.HTML.TagStream.Types , Text.HTML.TagStream.Utils , Text.HTML.TagStream.Entities Build-depends: base >= 4 && < 5 , bytestring , text , case-insensitive , transformers >= 0.2 , conduit >= 1.2 , conduit-extra >= 1.1.0 , resourcet , attoparsec >= 0.10 , blaze-builder , xml-conduit >= 1.1.0.0 , data-default >= 0.5.0 test-suite test hs-source-dirs: tests main-is: Tests.hs type: exitcode-stdio-1.0 build-depends: base >= 4 && < 5 , bytestring , text , conduit >= 1.2 , QuickCheck , HUnit , hspec >= 1.3 , tagstream-conduit , resourcet ghc-options: -Wall tagstream-conduit-0.5.5.3/LICENSE0000644000000000000000000000274712402232134014512 0ustar0000000000000000Copyright (c)2011, yihuang All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of yihuang nor the names of other contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. tagstream-conduit-0.5.5.3/HighlightText.hs0000644000000000000000000000255512402232134016612 0ustar0000000000000000import System.IO import System.Environment import System.Console.ANSI import Data.Maybe import Data.Text (Text) import qualified Data.Text as T import Data.Text.Encoding (decodeUtf8With) import Data.Text.Encoding.Error (lenientDecode) import qualified Data.Text.Lazy as L import qualified Data.Text.Lazy.Encoding as L import Data.Text.Lazy.Internal (defaultChunkSize) import Data.Text.Lazy.Builder (toLazyTextWith) import qualified Data.Conduit.Blaze as B import qualified Blaze.ByteString.Builder as B import qualified Blaze.ByteString.Builder.Char8 as B import qualified Data.Conduit as C import qualified Data.Conduit.Binary as CB import qualified Data.Conduit.List as CL import qualified Text.HTML.TagStream.Text as T color :: Color -> Text -> Text color c s = T.concat [ T.pack $ setSGRCode [SetColor Foreground Dull c] , s , T.pack $ setSGRCode [SetColor Foreground Dull White] ] main :: IO () main = do args <- getArgs filename <- maybe (fail "Highlight file") return (listToMaybe args) C.runResourceT $ CB.sourceFile filename C.$= CL.map (decodeUtf8With lenientDecode) C.$= T.tokenStream C.$= CL.map (B.fromLazyByteString . L.encodeUtf8 . (toLazyTextWith defaultChunkSize) . T.showToken (color Red)) C.$= B.builderToByteString C.$$ CB.sinkHandle stdout tagstream-conduit-0.5.5.3/ParseText.hs0000644000000000000000000000113112402232134015742 0ustar0000000000000000import Data.Maybe import System.Environment import qualified Text.HTML.TagStream.Text as T import Data.Text.Encoding (decodeUtf8With) import Data.Text.Encoding.Error (lenientDecode) import qualified Data.Conduit as C import qualified Data.Conduit.Binary as C import qualified Data.Conduit.List as CL main :: IO () main = do args <- getArgs filename <- maybe (fail "pass file path") return (listToMaybe args) _ <- C.runResourceT $ C.sourceFile filename C.$= CL.map (decodeUtf8With lenientDecode) C.$= T.tokenStream C.$$ CL.consume return () tagstream-conduit-0.5.5.3/Setup.hs0000644000000000000000000000005612402232134015130 0ustar0000000000000000import Distribution.Simple main = defaultMain tagstream-conduit-0.5.5.3/Highlight.hs0000644000000000000000000000165412402232134015744 0ustar0000000000000000import System.IO import System.Environment import System.Console.ANSI import Data.Maybe import Data.ByteString (ByteString) import qualified Data.ByteString.Char8 as S import Data.Conduit.Blaze (builderToByteString) import qualified Data.Conduit as C import qualified Data.Conduit.Binary as CB import qualified Data.Conduit.List as CL import qualified Text.HTML.TagStream.ByteString as S color :: Color -> ByteString -> ByteString color c s = S.concat [ S.pack $ setSGRCode [SetColor Foreground Dull c] , s , S.pack $ setSGRCode [SetColor Foreground Dull White] ] main :: IO () main = do args <- getArgs filename <- maybe (fail "Highlight file") return (listToMaybe args) C.runResourceT $ CB.sourceFile filename C.$= S.tokenStream C.$= CL.map (S.showToken (color Red)) C.$= builderToByteString C.$$ CB.sinkHandle stdout tagstream-conduit-0.5.5.3/Parse.hs0000644000000000000000000000071612402232134015105 0ustar0000000000000000import Data.Maybe import System.Environment import qualified Text.HTML.TagStream.ByteString as S import qualified Data.Conduit as C import qualified Data.Conduit.Binary as C import qualified Data.Conduit.List as CL main :: IO () main = do args <- getArgs filename <- maybe (fail "pass file path") return (listToMaybe args) _ <- C.runResourceT $ C.sourceFile filename C.$= S.tokenStream C.$$ CL.consume return () tagstream-conduit-0.5.5.3/tests/0000755000000000000000000000000012402232134014635 5ustar0000000000000000tagstream-conduit-0.5.5.3/tests/Tests.hs0000644000000000000000000002103512402232134016274 0ustar0000000000000000{-# LANGUAGE FlexibleInstances, OverloadedStrings, ViewPatterns #-} {-# OPTIONS_GHC -fno-warn-orphans #-} module Main where import Control.Applicative import Data.Monoid (Monoid(..),(<>)) import Data.ByteString (ByteString) import qualified Data.ByteString.Char8 as S import Data.Text (Text) import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Data.Conduit as C import qualified Control.Monad.Trans.Resource as C import qualified Data.Conduit.List as CL import Test.Hspec import Test.Hspec.QuickCheck (prop) import Test.HUnit hiding (Test) import Test.QuickCheck import Text.HTML.TagStream import qualified Text.HTML.TagStream.ByteString as S import qualified Text.HTML.TagStream.Text as T main :: IO () main = hspec $ do describe "[ByteString] Property" $ do prop "Text nodes can't be empty" propTextNotEmpty prop "Parse results can't empty" propResultNotEmpty describe "[Text] Property" $ do prop "Text nodes can't be empty" propTextNotEmptyText prop "Parse results can't empty" propResultNotEmptyText describe "[ByteString]One pass parse" onePassTests describe "[ByteString]Streamline parse" streamlineTests describe "[Text]One pass parse" onePassTestsText describe "[Text]Streamline parse" streamlineTestsText propTextNotEmpty :: ByteString -> Bool propTextNotEmpty = either (const False) text_not_empty . S.decode where text_not_empty = all not_empty not_empty (Text s) = S.length s > 0 not_empty _ = True propResultNotEmpty :: ByteString -> Bool propResultNotEmpty s = either (const False) not_empty . S.decode $ s where not_empty tokens = (S.null s && null tokens) || (not (S.null s) && not (null tokens)) propTextNotEmptyText :: Text -> Bool propTextNotEmptyText = either (const False) text_not_empty . T.decode where text_not_empty = all not_empty not_empty (Text s) = not (T.null s) not_empty _ = True propResultNotEmptyText :: Text -> Bool propResultNotEmptyText s = either (const False) not_empty . T.decode $ s where not_empty tokens = (T.null s && null tokens) || (not (T.null s) && not (null tokens)) encodeTokenUtf8 :: Token' Text -> Token' ByteString encodeTokenUtf8 = fmap T.encodeUtf8 onePassTests :: Spec onePassTests = mapM_ one testcases where one (T.encodeUtf8 -> str, map encodeTokenUtf8 -> tokens) = it (S.unpack str) $ do result <- combineText <$> assertDecodeBS str assertEqual "one-pass parse result incorrect" tokens result onePassTestsText :: Spec onePassTestsText = mapM_ one testcases where one (str, tokens) = it (T.unpack str) $ do result <- combineText <$> assertDecodeText str assertEqual "one-pass parse result incorrect" tokens result streamlineTests :: Spec streamlineTests = mapM_ one testcases where isIncomplete (Incomplete _) = True isIncomplete _ = False one (T.encodeUtf8 -> str, map encodeTokenUtf8 -> tokens) = it (S.unpack str) $ do -- streamline parse result don't contain the trailing Incomplete token. let tokens' = reverse . dropWhile isIncomplete . reverse $ tokens result <- combineText <$> C.runResourceT ( CL.sourceList (map S.singleton (S.unpack str)) C.$= S.tokenStream C.$$ CL.consume ) assertEqual "streamline parse result incorrect" tokens' result streamlineTestsText :: Spec streamlineTestsText = mapM_ one testcases where isIncomplete (Incomplete _) = True isIncomplete _ = False one (T.encodeUtf8 -> str, tokens) = it (S.unpack str) $ do -- streamline parse result don't contain the trailing Incomplete token. let tokens' = reverse . dropWhile isIncomplete . reverse $ tokens result <- combineText <$> C.runResourceT ( CL.sourceList (map S.singleton (S.unpack str)) C.$= T.tokenStreamBS C.$$ CL.consume ) assertEqual "streamline parse result incorrect" tokens' result testcases :: [(Text, [Token' Text])] testcases = -- attributes {{{ [ ( "